1 /*- 2 * Copyright (c) 2023 NVIDIA corporation & affiliates. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include "opt_ipsec.h" 28 29 #include <sys/types.h> 30 #include <netinet/in.h> 31 #include <sys/socket.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <net/pfkeyv2.h> 35 #include <netipsec/key_var.h> 36 #include <netipsec/keydb.h> 37 #include <netipsec/ipsec.h> 38 #include <netipsec/xform.h> 39 #include <netipsec/ipsec_offload.h> 40 #include <dev/mlx5/fs.h> 41 #include <dev/mlx5/mlx5_en/en.h> 42 #include <dev/mlx5/qp.h> 43 #include <dev/mlx5/mlx5_accel/ipsec.h> 44 #include <dev/mlx5/mlx5_core/fs_core.h> 45 #include <dev/mlx5/mlx5_core/fs_chains.h> 46 47 /* 48 * TX tables are organized differently for Ethernet and for RoCE: 49 * 50 * +=========+ 51 * Ethernet Tx | SA KSPI | match 52 * --------------------->|Flowtable|----->+ + 53 * | |\ | / \ 54 * +=========+ | | / \ +=========+ +=========+ 55 * miss | | / \ | Status | | | 56 * DROP<--------+ |---->|Encrypt|------>|Flowtable|---->| TX NS | 57 * | \ / | | | | 58 * | \ / +=========+ +=========+ 59 * +=========+ +=========+ | \ / | 60 * RoCE | Policy | match|SA ReqId |match| + | 61 * Tx |Flowtable|----->|Flowtable|---->+ | 62 * ---->|IP header| |ReqId+IP | | 63 * | | | header |--------------------------------+ 64 * +=========+ +=========+ miss | 65 * | | 66 * | miss | 67 * +------------------------------------------------------- 68 * 69 * +=========+ 70 * | RDMA | 71 * |Flowtable| 72 * | | 73 * Rx Tables and rules: +=========+ 74 * + / 75 * +=========+ +=========+ / \ +=========+ +=========+ /match 76 * | Policy | | SA | / \ | Status | | RoCE |/ 77 * ---->|Flowtable| match|Flowtable| match / \ |Flowtable|----->|Flowtable| 78 * |IP header|----->|IP header|----->|Decrypt|----->| | | Roce V2 | 79 * | | |+ESP+SPI | \ / | | | UDP port|\ 80 * +=========+ +=========+ \ / +=========+ +=========+ \miss 81 * | | \ / \ 82 * | | + +=========+ 83 * | miss | miss | Ethernet| 84 * +--------------->---------------------------------------------------->| RX NS | 85 * | | 86 * +=========+ 87 * 88 */ 89 90 #define NUM_IPSEC_FTE BIT(15) 91 #define IPSEC_TUNNEL_DEFAULT_TTL 0x40 92 93 struct mlx5e_ipsec_fc { 94 struct mlx5_fc *cnt; 95 struct mlx5_fc *drop; 96 }; 97 98 struct mlx5e_ipsec_ft { 99 struct mutex mutex; /* Protect changes to this struct */ 100 struct mlx5_flow_table *pol; 101 struct mlx5_flow_table *sa_kspi; 102 struct mlx5_flow_table *sa; 103 struct mlx5_flow_table *status; 104 u32 refcnt; 105 }; 106 107 struct mlx5e_ipsec_tx_roce { 108 struct mlx5_flow_group *g; 109 struct mlx5_flow_table *ft; 110 struct mlx5_flow_handle *rule; 111 struct mlx5_flow_namespace *ns; 112 }; 113 114 struct mlx5e_ipsec_miss { 115 struct mlx5_flow_group *group; 116 struct mlx5_flow_handle *rule; 117 }; 118 119 struct mlx5e_ipsec_tx { 120 struct mlx5e_ipsec_ft ft; 121 struct mlx5e_ipsec_miss pol; 122 struct mlx5e_ipsec_miss kspi_miss; 123 struct mlx5e_ipsec_rule status; 124 struct mlx5e_ipsec_rule kspi_bypass_rule; /*rule for IPSEC bypass*/ 125 struct mlx5_flow_namespace *ns; 126 struct mlx5e_ipsec_fc *fc; 127 struct mlx5_fs_chains *chains; 128 struct mlx5e_ipsec_tx_roce roce; 129 }; 130 131 struct mlx5e_ipsec_rx_roce { 132 struct mlx5_flow_group *g; 133 struct mlx5_flow_table *ft; 134 struct mlx5_flow_handle *rule; 135 struct mlx5e_ipsec_miss roce_miss; 136 137 struct mlx5_flow_table *ft_rdma; 138 struct mlx5_flow_namespace *ns_rdma; 139 }; 140 141 struct mlx5e_ipsec_rx { 142 struct mlx5e_ipsec_ft ft; 143 struct mlx5e_ipsec_miss pol; 144 struct mlx5e_ipsec_miss sa; 145 struct mlx5e_ipsec_rule status; 146 struct mlx5_flow_namespace *ns; 147 struct mlx5e_ipsec_fc *fc; 148 struct mlx5_fs_chains *chains; 149 struct mlx5e_ipsec_rx_roce roce; 150 }; 151 152 static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec, 153 u16 kspi); 154 static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec); 155 156 static void setup_fte_no_frags(struct mlx5_flow_spec *spec) 157 { 158 /* Non fragmented */ 159 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 160 161 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); 162 MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); 163 } 164 165 static void setup_fte_esp(struct mlx5_flow_spec *spec) 166 { 167 /* ESP header */ 168 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 169 170 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 171 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); 172 } 173 174 static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap) 175 { 176 /* SPI number */ 177 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 178 179 if (encap) { 180 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.inner_esp_spi); 181 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.inner_esp_spi, spi); 182 } else { 183 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); 184 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); 185 } 186 } 187 188 static void 189 setup_fte_vid(struct mlx5_flow_spec *spec, u16 vid) 190 { 191 /* virtual lan tag */ 192 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 193 194 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 195 outer_headers.cvlan_tag); 196 MLX5_SET(fte_match_param, spec->match_value, 197 outer_headers.cvlan_tag, 1); 198 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 199 outer_headers.first_vid); 200 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, 201 vid); 202 } 203 204 static void 205 clear_fte_vid(struct mlx5_flow_spec *spec) 206 { 207 MLX5_SET(fte_match_param, spec->match_criteria, 208 outer_headers.cvlan_tag, 0); 209 MLX5_SET(fte_match_param, spec->match_value, 210 outer_headers.cvlan_tag, 0); 211 MLX5_SET(fte_match_param, spec->match_criteria, 212 outer_headers.first_vid, 0); 213 MLX5_SET(fte_match_param, spec->match_value, 214 outer_headers.first_vid, 0); 215 } 216 217 static void 218 setup_fte_no_vid(struct mlx5_flow_spec *spec) 219 { 220 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 221 outer_headers.cvlan_tag); 222 MLX5_SET(fte_match_param, spec->match_value, 223 outer_headers.cvlan_tag, 0); 224 } 225 226 static struct mlx5_fs_chains * 227 ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, 228 enum mlx5_flow_namespace_type ns, int base_prio, 229 int base_level, struct mlx5_flow_table **root_ft) 230 { 231 struct mlx5_chains_attr attr = {}; 232 struct mlx5_fs_chains *chains; 233 struct mlx5_flow_table *ft; 234 int err; 235 236 attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED | 237 MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 238 attr.max_grp_num = 2; 239 attr.default_ft = miss_ft; 240 attr.ns = ns; 241 attr.fs_base_prio = base_prio; 242 attr.fs_base_level = base_level; 243 chains = mlx5_chains_create(mdev, &attr); 244 if (IS_ERR(chains)) 245 return chains; 246 247 /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */ 248 ft = mlx5_chains_get_table(chains, 0, 1, 0); 249 if (IS_ERR(ft)) { 250 err = PTR_ERR(ft); 251 goto err_chains_get; 252 } 253 254 *root_ft = ft; 255 return chains; 256 257 err_chains_get: 258 mlx5_chains_destroy(chains); 259 return ERR_PTR(err); 260 } 261 262 static void ipsec_chains_destroy(struct mlx5_fs_chains *chains) 263 { 264 mlx5_chains_put_table(chains, 0, 1, 0); 265 mlx5_chains_destroy(chains); 266 } 267 268 static struct mlx5_flow_table * 269 ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio) 270 { 271 return mlx5_chains_get_table(chains, 0, prio + 1, 0); 272 } 273 274 static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio) 275 { 276 mlx5_chains_put_table(chains, 0, prio + 1, 0); 277 } 278 279 static struct mlx5_flow_table *ipsec_rx_ft_create(struct mlx5_flow_namespace *ns, 280 int level, int prio, 281 int max_num_groups) 282 { 283 struct mlx5_flow_table_attr ft_attr = {}; 284 285 ft_attr.max_fte = NUM_IPSEC_FTE; 286 ft_attr.level = level; 287 ft_attr.prio = prio; 288 ft_attr.autogroup.max_num_groups = max_num_groups; 289 ft_attr.autogroup.num_reserved_entries = 1; 290 291 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 292 } 293 294 static int ipsec_miss_create(struct mlx5_core_dev *mdev, 295 struct mlx5_flow_table *ft, 296 struct mlx5e_ipsec_miss *miss, 297 struct mlx5_flow_destination *dest) 298 { 299 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 300 struct mlx5_flow_act flow_act = {}; 301 struct mlx5_flow_spec *spec; 302 u32 *flow_group_in; 303 int err = 0; 304 305 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 306 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 307 if (!flow_group_in || !spec) { 308 err = -ENOMEM; 309 goto out; 310 } 311 312 /* Create miss_group */ 313 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 314 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 315 miss->group = mlx5_create_flow_group(ft, flow_group_in); 316 if (IS_ERR(miss->group)) { 317 err = PTR_ERR(miss->group); 318 mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", 319 err); 320 goto out; 321 } 322 323 if (dest) 324 flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST; 325 else 326 flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP; 327 /* Create miss rule */ 328 miss->rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1); 329 if (IS_ERR(miss->rule)) { 330 mlx5_destroy_flow_group(miss->group); 331 err = PTR_ERR(miss->rule); 332 mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", 333 err); 334 goto out; 335 } 336 out: 337 kvfree(flow_group_in); 338 kvfree(spec); 339 return err; 340 } 341 342 static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, 343 struct mlx5_flow_act *flow_act) 344 { 345 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 346 enum mlx5_flow_namespace_type ns_type; 347 struct mlx5_modify_hdr *modify_hdr; 348 349 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 350 switch (dir) { 351 case IPSEC_DIR_INBOUND: 352 MLX5_SET(set_action_in, action, field, 353 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 354 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 355 break; 356 case IPSEC_DIR_OUTBOUND: 357 MLX5_SET(set_action_in, action, field, 358 MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); 359 ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 360 break; 361 default: 362 return -EINVAL; 363 } 364 365 MLX5_SET(set_action_in, action, data, val); 366 MLX5_SET(set_action_in, action, offset, 0); 367 MLX5_SET(set_action_in, action, length, 32); 368 369 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); 370 if (IS_ERR(modify_hdr)) { 371 mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", 372 PTR_ERR(modify_hdr)); 373 return PTR_ERR(modify_hdr); 374 } 375 376 flow_act->modify_hdr = modify_hdr; 377 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 378 return 0; 379 } 380 381 static int 382 setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, 383 struct mlx5_pkt_reformat_params *reformat_params) 384 { 385 struct udphdr *udphdr; 386 size_t bfflen = 16; 387 char *reformatbf; 388 __be32 spi; 389 void *hdr; 390 391 if (attrs->family == AF_INET) 392 if (attrs->encap) 393 reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4; 394 else 395 reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; 396 else 397 reformat_params->type = 398 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; 399 400 if (attrs->encap) 401 bfflen += sizeof(*udphdr); 402 reformatbf = kzalloc(bfflen, GFP_KERNEL); 403 if (!reformatbf) 404 return -ENOMEM; 405 406 hdr = reformatbf; 407 if (attrs->encap) { 408 udphdr = (struct udphdr *)reformatbf; 409 udphdr->uh_sport = attrs->sport; 410 udphdr->uh_dport = attrs->dport; 411 hdr += sizeof(*udphdr); 412 } 413 414 /* convert to network format */ 415 spi = htonl(attrs->spi); 416 memcpy(hdr, &spi, 4); 417 418 reformat_params->param_0 = attrs->authsize; 419 reformat_params->size = bfflen; 420 reformat_params->data = reformatbf; 421 422 return 0; 423 } 424 425 static int setup_pkt_reformat(struct mlx5_core_dev *mdev, 426 struct mlx5_accel_esp_xfrm_attrs *attrs, 427 struct mlx5_flow_act *flow_act) 428 { 429 enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 430 struct mlx5_pkt_reformat_params reformat_params = {}; 431 struct mlx5_pkt_reformat *pkt_reformat; 432 int ret; 433 434 if (attrs->dir == IPSEC_DIR_INBOUND) { 435 if (attrs->encap) 436 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP; 437 else 438 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; 439 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 440 goto cmd; 441 } 442 443 ret = setup_pkt_transport_reformat(attrs, &reformat_params); 444 if (ret) 445 return ret; 446 cmd: 447 pkt_reformat = 448 mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); 449 if (reformat_params.data) 450 kfree(reformat_params.data); 451 if (IS_ERR(pkt_reformat)) 452 return PTR_ERR(pkt_reformat); 453 454 flow_act->pkt_reformat = pkt_reformat; 455 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 456 return 0; 457 } 458 459 static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, 460 __be32 *daddr) 461 { 462 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 463 464 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 465 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 466 467 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 468 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); 469 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 470 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); 471 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 472 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 473 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 474 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 475 } 476 477 static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, 478 __be32 *daddr) 479 { 480 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 481 482 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 483 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 484 485 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 486 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); 487 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 488 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); 489 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 490 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); 491 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 492 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); 493 } 494 495 static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 496 { 497 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 498 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 499 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 500 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 501 struct mlx5_flow_destination dest[2] = {}; 502 struct mlx5_flow_act flow_act = {}; 503 struct mlx5_flow_handle *rule; 504 struct mlx5_flow_spec *spec; 505 struct mlx5e_ipsec_rx *rx; 506 struct mlx5_fc *counter; 507 int err; 508 509 rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6; 510 511 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 512 if (!spec) 513 return -ENOMEM; 514 515 if (!attrs->drop) { 516 err = setup_modify_header(mdev, sa_entry->kspi | BIT(31), IPSEC_DIR_INBOUND, 517 &flow_act); 518 if (err) 519 goto err_mod_header; 520 } 521 522 err = setup_pkt_reformat(mdev, attrs, &flow_act); 523 if (err) 524 goto err_pkt_reformat; 525 526 counter = mlx5_fc_create(mdev, false); 527 if (IS_ERR(counter)) { 528 err = PTR_ERR(counter); 529 goto err_add_cnt; 530 } 531 532 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 533 flow_act.crypto.op = MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT; 534 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 535 flow_act.flags |= FLOW_ACT_NO_APPEND; 536 537 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 538 MLX5_FLOW_CONTEXT_ACTION_COUNT; 539 540 if (attrs->drop) 541 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 542 else 543 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 544 545 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 546 dest[0].ft = rx->ft.status; 547 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 548 dest[1].counter_id = mlx5_fc_id(counter); 549 550 if (attrs->family == AF_INET) 551 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 552 else 553 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 554 555 if (!attrs->encap) 556 setup_fte_esp(spec); 557 558 setup_fte_spi(spec, attrs->spi, attrs->encap); 559 setup_fte_no_frags(spec); 560 561 if (sa_entry->vid != VLAN_NONE) 562 setup_fte_vid(spec, sa_entry->vid); 563 else 564 setup_fte_no_vid(spec); 565 566 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 567 if (IS_ERR(rule)) { 568 err = PTR_ERR(rule); 569 mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); 570 goto err_add_flow; 571 } 572 ipsec_rule->rule = rule; 573 574 /* Add another rule for zero vid */ 575 if (sa_entry->vid == VLAN_NONE) { 576 clear_fte_vid(spec); 577 setup_fte_vid(spec, 0); 578 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 579 if (IS_ERR(rule)) { 580 err = PTR_ERR(rule); 581 mlx5_core_err(mdev, 582 "fail to add RX ipsec zero vid rule err=%d\n", 583 err); 584 goto err_add_flow; 585 } 586 ipsec_rule->vid_zero_rule = rule; 587 } 588 589 kvfree(spec); 590 ipsec_rule->fc = counter; 591 ipsec_rule->modify_hdr = flow_act.modify_hdr; 592 ipsec_rule->pkt_reformat = flow_act.pkt_reformat; 593 return 0; 594 595 err_add_flow: 596 mlx5_fc_destroy(mdev, counter); 597 if (ipsec_rule->rule != NULL) 598 mlx5_del_flow_rules(&ipsec_rule->rule); 599 err_add_cnt: 600 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 601 err_pkt_reformat: 602 if (flow_act.modify_hdr != NULL) 603 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 604 err_mod_header: 605 kvfree(spec); 606 607 return err; 608 } 609 610 static struct mlx5_flow_table *ipsec_tx_ft_create(struct mlx5_flow_namespace *ns, 611 int level, int prio, 612 int max_num_groups) 613 { 614 struct mlx5_flow_table_attr ft_attr = {}; 615 616 ft_attr.autogroup.num_reserved_entries = 1; 617 ft_attr.autogroup.max_num_groups = max_num_groups; 618 ft_attr.max_fte = NUM_IPSEC_FTE; 619 ft_attr.level = level; 620 ft_attr.prio = prio; 621 622 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 623 } 624 625 static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 626 { 627 struct mlx5_flow_destination dest = {}; 628 struct mlx5_flow_act flow_act = {}; 629 struct mlx5_flow_handle *fte; 630 int err; 631 632 /* create fte */ 633 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_COUNT | 634 MLX5_FLOW_CONTEXT_ACTION_ALLOW; 635 636 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 637 dest.counter_id = mlx5_fc_id(tx->fc->cnt); 638 fte = mlx5_add_flow_rules(tx->ft.status, NULL, &flow_act, &dest, 1); 639 if (IS_ERR_OR_NULL(fte)) { 640 err = PTR_ERR(fte); 641 mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err); 642 goto err_rule; 643 } 644 645 tx->status.rule = fte; 646 return 0; 647 648 err_rule: 649 return err; 650 } 651 652 static void tx_destroy_roce(struct mlx5e_ipsec_tx *tx) { 653 if (!tx->roce.ft) 654 return; 655 656 mlx5_del_flow_rules(&tx->roce.rule); 657 mlx5_destroy_flow_group(tx->roce.g); 658 mlx5_destroy_flow_table(tx->roce.ft); 659 tx->roce.ft = NULL; 660 } 661 662 /* IPsec TX flow steering */ 663 static void tx_destroy(struct mlx5e_ipsec_tx *tx) 664 { 665 tx_destroy_roce(tx); 666 if (tx->chains) { 667 ipsec_chains_destroy(tx->chains); 668 } else { 669 mlx5_del_flow_rules(&tx->pol.rule); 670 mlx5_destroy_flow_group(tx->pol.group); 671 mlx5_destroy_flow_table(tx->ft.pol); 672 } 673 mlx5_destroy_flow_table(tx->ft.sa); 674 mlx5_del_flow_rules(&tx->kspi_miss.rule); 675 mlx5_destroy_flow_group(tx->kspi_miss.group); 676 mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule); 677 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 678 mlx5_destroy_flow_table(tx->ft.sa_kspi); 679 mlx5_del_flow_rules(&tx->status.rule); 680 mlx5_destroy_flow_table(tx->ft.status); 681 } 682 683 static int ipsec_tx_roce_rule_setup(struct mlx5_core_dev *mdev, 684 struct mlx5e_ipsec_tx *tx) 685 { 686 struct mlx5_flow_destination dst = {}; 687 struct mlx5_flow_act flow_act = {}; 688 struct mlx5_flow_handle *rule; 689 int err = 0; 690 691 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 692 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 693 dst.ft = tx->ft.pol; 694 rule = mlx5_add_flow_rules(tx->roce.ft, NULL, &flow_act, &dst, 1); 695 if (IS_ERR(rule)) { 696 err = PTR_ERR(rule); 697 mlx5_core_err(mdev, "Fail to add TX roce ipsec rule err=%d\n", 698 err); 699 goto out; 700 } 701 tx->roce.rule = rule; 702 703 out: 704 return err; 705 } 706 707 static int ipsec_tx_create_roce(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 708 { 709 struct mlx5_flow_table_attr ft_attr = {}; 710 struct mlx5_flow_table *ft; 711 struct mlx5_flow_group *g; 712 int ix = 0; 713 int err; 714 u32 *in; 715 716 if (!tx->roce.ns) 717 return -EOPNOTSUPP; 718 719 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 720 if (!in) 721 return -ENOMEM; 722 723 ft_attr.max_fte = 1; 724 ft = mlx5_create_flow_table(tx->roce.ns, &ft_attr); 725 if (IS_ERR(ft)) { 726 err = PTR_ERR(ft); 727 mlx5_core_err(mdev, "Fail to create ipsec tx roce ft err=%d\n", 728 err); 729 goto fail_table; 730 } 731 tx->roce.ft = ft; 732 733 MLX5_SET_CFG(in, start_flow_index, ix); 734 ix += 1; 735 MLX5_SET_CFG(in, end_flow_index, ix - 1); 736 g = mlx5_create_flow_group(ft, in); 737 if (IS_ERR(g)) { 738 err = PTR_ERR(g); 739 mlx5_core_err(mdev, "Fail to create ipsec tx roce group err=%d\n", 740 err); 741 goto fail_group; 742 } 743 tx->roce.g = g; 744 745 err = ipsec_tx_roce_rule_setup(mdev, tx); 746 if (err) { 747 mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err); 748 goto fail_rule; 749 } 750 751 kvfree(in); 752 return 0; 753 754 fail_rule: 755 mlx5_destroy_flow_group(tx->roce.g); 756 fail_group: 757 mlx5_destroy_flow_table(tx->roce.ft); 758 tx->roce.ft = NULL; 759 fail_table: 760 kvfree(in); 761 return err; 762 } 763 764 /* 765 * Setting a rule in KSPI table for values that should bypass IPSEC. 766 * 767 * mdev - mlx5 core device 768 * tx - IPSEC TX 769 * return - 0 for success errno for failure 770 */ 771 static int tx_create_kspi_bypass_rules(struct mlx5_core_dev *mdev, 772 struct mlx5e_ipsec_tx *tx) 773 { 774 struct mlx5_flow_destination dest = {}; 775 struct mlx5_flow_act flow_act = {}; 776 struct mlx5_flow_act flow_act_kspi = {}; 777 struct mlx5_flow_handle *rule; 778 struct mlx5_flow_spec *spec; 779 int err; 780 781 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 782 if (!spec) 783 return -ENOMEM; 784 785 dest.ft = tx->ft.status; 786 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 787 flow_act_kspi.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 788 789 setup_fte_reg_a_with_tag(spec, IPSEC_ACCEL_DRV_SPI_BYPASS); 790 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act_kspi, 791 &dest, 1); 792 if (IS_ERR(rule)) { 793 err = PTR_ERR(rule); 794 mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", 795 err); 796 goto err_add_kspi_rule; 797 } 798 tx->kspi_bypass_rule.kspi_rule = rule; 799 800 /* set the rule for packets withoiut ipsec tag. */ 801 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 802 memset(spec, 0, sizeof(*spec)); 803 setup_fte_reg_a_no_tag(spec); 804 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act, &dest, 1); 805 if (IS_ERR(rule)) { 806 err = PTR_ERR(rule); 807 mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", err); 808 goto err_add_rule; 809 } 810 tx->kspi_bypass_rule.rule = rule; 811 812 kvfree(spec); 813 return 0; 814 err_add_rule: 815 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 816 err_add_kspi_rule: 817 kvfree(spec); 818 return err; 819 } 820 821 822 static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 823 { 824 struct mlx5_flow_destination dest = {}; 825 struct mlx5_flow_table *ft; 826 int err; 827 828 /* 829 * Tx flow is different for ethernet traffic then for RoCE packets 830 * For Ethernet packets we start in SA KSPI table that matches KSPI of SA rule 831 * to the KSPI in the packet metadata 832 * For RoCE traffic we start in Policy table, then move to SA table 833 * which matches either reqid of the SA rule to reqid reported by policy table 834 * or ip header fields of SA to the packet IP header fields. 835 * Tables are ordered by their level so we set kspi 836 * with level 0 to have it first one for ethernet traffic. 837 * For RoCE the RoCE TX table direct the packets to policy table explicitly 838 */ 839 ft = ipsec_tx_ft_create(tx->ns, 0, 0, 4); 840 if (IS_ERR(ft)) 841 return PTR_ERR(ft); 842 tx->ft.sa_kspi = ft; 843 844 ft = ipsec_tx_ft_create(tx->ns, 2, 0, 4); 845 if (IS_ERR(ft)) { 846 err = PTR_ERR(ft); 847 goto err_reqid_ft; 848 } 849 tx->ft.sa = ft; 850 851 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 852 tx->chains = ipsec_chains_create( 853 mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 1, 854 &tx->ft.pol); 855 if (IS_ERR(tx->chains)) { 856 err = PTR_ERR(tx->chains); 857 goto err_pol_ft; 858 } 859 } else { 860 ft = ipsec_tx_ft_create(tx->ns, 1, 0, 2); 861 if (IS_ERR(ft)) { 862 err = PTR_ERR(ft); 863 goto err_pol_ft; 864 } 865 tx->ft.pol = ft; 866 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 867 dest.ft = tx->ft.sa; 868 err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); 869 if (err) 870 goto err_pol_miss; 871 } 872 873 ft = ipsec_tx_ft_create(tx->ns, 2, 0, 1); 874 if (IS_ERR(ft)) { 875 err = PTR_ERR(ft); 876 goto err_status_ft; 877 } 878 tx->ft.status = ft; 879 880 /* set miss rule for kspi table with drop action*/ 881 err = ipsec_miss_create(mdev, tx->ft.sa_kspi, &tx->kspi_miss, NULL); 882 if (err) 883 goto err_kspi_miss; 884 885 err = tx_create_kspi_bypass_rules(mdev, tx); 886 if (err) 887 goto err_kspi_rule; 888 889 err = ipsec_counter_rule_tx(mdev, tx); 890 if (err) 891 goto err_status_rule; 892 893 err = ipsec_tx_create_roce(mdev, tx); 894 if (err) 895 goto err_counter_rule; 896 897 return 0; 898 899 err_counter_rule: 900 mlx5_del_flow_rules(&tx->status.rule); 901 err_status_rule: 902 mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule); 903 mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule); 904 err_kspi_rule: 905 mlx5_destroy_flow_table(tx->ft.status); 906 err_status_ft: 907 if (tx->chains) { 908 ipsec_chains_destroy(tx->chains); 909 } else { 910 mlx5_del_flow_rules(&tx->pol.rule); 911 mlx5_destroy_flow_group(tx->pol.group); 912 } 913 err_pol_miss: 914 if (!tx->chains) 915 mlx5_destroy_flow_table(tx->ft.pol); 916 err_pol_ft: 917 mlx5_del_flow_rules(&tx->kspi_miss.rule); 918 mlx5_destroy_flow_group(tx->kspi_miss.group); 919 err_kspi_miss: 920 mlx5_destroy_flow_table(tx->ft.sa); 921 err_reqid_ft: 922 mlx5_destroy_flow_table(tx->ft.sa_kspi); 923 return err; 924 } 925 926 static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 927 struct mlx5e_ipsec_tx *tx) 928 { 929 int err; 930 931 if (tx->ft.refcnt) 932 goto skip; 933 934 err = tx_create(mdev, tx); 935 if (err) 936 return err; 937 938 skip: 939 tx->ft.refcnt++; 940 return 0; 941 } 942 943 static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) 944 { 945 if (--tx->ft.refcnt) 946 return; 947 948 tx_destroy(tx); 949 } 950 951 static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, 952 struct mlx5e_ipsec *ipsec) 953 { 954 struct mlx5e_ipsec_tx *tx = ipsec->tx; 955 int err; 956 957 mutex_lock(&tx->ft.mutex); 958 err = tx_get(mdev, ipsec, tx); 959 mutex_unlock(&tx->ft.mutex); 960 if (err) 961 return ERR_PTR(err); 962 963 return tx; 964 } 965 966 static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, 967 struct mlx5e_ipsec *ipsec, 968 u32 prio) 969 { 970 struct mlx5e_ipsec_tx *tx = ipsec->tx; 971 struct mlx5_flow_table *ft; 972 int err; 973 974 mutex_lock(&tx->ft.mutex); 975 err = tx_get(mdev, ipsec, tx); 976 if (err) 977 goto err_get; 978 979 ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol; 980 if (IS_ERR(ft)) { 981 err = PTR_ERR(ft); 982 goto err_get_ft; 983 } 984 985 mutex_unlock(&tx->ft.mutex); 986 return ft; 987 988 err_get_ft: 989 tx_put(ipsec, tx); 990 err_get: 991 mutex_unlock(&tx->ft.mutex); 992 return ERR_PTR(err); 993 } 994 995 static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio) 996 { 997 struct mlx5e_ipsec_tx *tx = ipsec->tx; 998 999 mutex_lock(&tx->ft.mutex); 1000 if (tx->chains) 1001 ipsec_chains_put_table(tx->chains, prio); 1002 1003 tx_put(ipsec, tx); 1004 mutex_unlock(&tx->ft.mutex); 1005 } 1006 1007 static void tx_ft_put(struct mlx5e_ipsec *ipsec) 1008 { 1009 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1010 1011 mutex_lock(&tx->ft.mutex); 1012 tx_put(ipsec, tx); 1013 mutex_unlock(&tx->ft.mutex); 1014 } 1015 1016 static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec, 1017 u16 kspi) 1018 { 1019 /* Add IPsec indicator in metadata_reg_a. */ 1020 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1021 1022 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1023 misc_parameters_2.metadata_reg_a); 1024 MLX5_SET(fte_match_param, spec->match_value, 1025 misc_parameters_2.metadata_reg_a, 1026 MLX5_ETH_WQE_FT_META_IPSEC << 23 | kspi); 1027 } 1028 1029 static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec) 1030 { 1031 /* Add IPsec indicator in metadata_reg_a. */ 1032 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1033 1034 MLX5_SET(fte_match_param, spec->match_criteria, 1035 misc_parameters_2.metadata_reg_a, 1036 MLX5_ETH_WQE_FT_META_IPSEC << 23); 1037 MLX5_SET(fte_match_param, spec->match_value, 1038 misc_parameters_2.metadata_reg_a, 1039 0); 1040 } 1041 1042 static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) 1043 { 1044 /* Pass policy check before choosing this SA */ 1045 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1046 1047 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1048 misc_parameters_2.metadata_reg_c_0); 1049 MLX5_SET(fte_match_param, spec->match_value, 1050 misc_parameters_2.metadata_reg_c_0, reqid); 1051 } 1052 1053 static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) 1054 { 1055 switch (upspec->proto) { 1056 case IPPROTO_UDP: 1057 if (upspec->dport) { 1058 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1059 spec->match_criteria, udp_dport); 1060 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1061 udp_dport, upspec->dport); 1062 } 1063 1064 if (upspec->sport) { 1065 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1066 spec->match_criteria, udp_sport); 1067 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1068 udp_dport, upspec->sport); 1069 } 1070 break; 1071 case IPPROTO_TCP: 1072 if (upspec->dport) { 1073 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1074 spec->match_criteria, tcp_dport); 1075 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1076 tcp_dport, upspec->dport); 1077 } 1078 1079 if (upspec->sport) { 1080 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, 1081 spec->match_criteria, tcp_sport); 1082 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1083 tcp_dport, upspec->sport); 1084 } 1085 break; 1086 default: 1087 return; 1088 } 1089 1090 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1091 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); 1092 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); 1093 } 1094 1095 static int tx_add_kspi_rule(struct mlx5e_ipsec_sa_entry *sa_entry, 1096 struct mlx5e_ipsec_tx *tx, 1097 struct mlx5_flow_act *flow_act, 1098 struct mlx5_flow_destination *dest, 1099 int num_dest) 1100 { 1101 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1102 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1103 struct mlx5_flow_handle *rule; 1104 struct mlx5_flow_spec *spec; 1105 int err; 1106 1107 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1108 if (!spec) 1109 return -ENOMEM; 1110 1111 setup_fte_no_frags(spec); 1112 setup_fte_reg_a_with_tag(spec, sa_entry->kspi); 1113 1114 rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, flow_act, dest, num_dest); 1115 if (IS_ERR(rule)) { 1116 err = PTR_ERR(rule); 1117 mlx5_core_err(mdev, "fail to add TX ipsec kspi rule err=%d\n", err); 1118 goto err_add_kspi_flow; 1119 } 1120 ipsec_rule->kspi_rule = rule; 1121 kvfree(spec); 1122 return 0; 1123 1124 err_add_kspi_flow: 1125 kvfree(spec); 1126 return err; 1127 } 1128 1129 static int tx_add_reqid_ip_rules(struct mlx5e_ipsec_sa_entry *sa_entry, 1130 struct mlx5e_ipsec_tx *tx, 1131 struct mlx5_flow_act *flow_act, 1132 struct mlx5_flow_destination *dest, 1133 int num_dest) 1134 { 1135 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1136 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1137 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1138 struct mlx5_flow_handle *rule; 1139 struct mlx5_flow_spec *spec; 1140 int err; 1141 1142 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1143 if (!spec) 1144 return -ENOMEM; 1145 1146 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 1147 1148 if(attrs->reqid) { 1149 setup_fte_no_frags(spec); 1150 setup_fte_reg_c0(spec, attrs->reqid); 1151 rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest); 1152 if (IS_ERR(rule)) { 1153 err = PTR_ERR(rule); 1154 mlx5_core_err(mdev, "fail to add TX ipsec reqid rule err=%d\n", err); 1155 goto err_add_reqid_rule; 1156 } 1157 ipsec_rule->reqid_rule = rule; 1158 memset(spec, 0, sizeof(*spec)); 1159 } 1160 1161 if (attrs->family == AF_INET) 1162 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1163 else 1164 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1165 setup_fte_no_frags(spec); 1166 1167 rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest); 1168 if (IS_ERR(rule)) { 1169 err = PTR_ERR(rule); 1170 mlx5_core_err(mdev, "fail to add TX ipsec ip rule err=%d\n", err); 1171 goto err_add_ip_rule; 1172 } 1173 ipsec_rule->rule = rule; 1174 kvfree(spec); 1175 return 0; 1176 1177 err_add_ip_rule: 1178 mlx5_del_flow_rules(&ipsec_rule->reqid_rule); 1179 err_add_reqid_rule: 1180 kvfree(spec); 1181 return err; 1182 } 1183 1184 static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1185 { 1186 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1187 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1188 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1189 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1190 struct mlx5_flow_destination dest[2] = {}; 1191 struct mlx5_flow_act flow_act = {}; 1192 struct mlx5e_ipsec_tx *tx; 1193 struct mlx5_fc *counter; 1194 int err; 1195 1196 tx = tx_ft_get(mdev, ipsec); 1197 if (IS_ERR(tx)) 1198 return PTR_ERR(tx); 1199 1200 err = setup_pkt_reformat(mdev, attrs, &flow_act); 1201 if (err) 1202 goto err_pkt_reformat; 1203 1204 counter = mlx5_fc_create(mdev, false); 1205 if (IS_ERR(counter)) { 1206 err = PTR_ERR(counter); 1207 goto err_add_cnt; 1208 } 1209 1210 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1211 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1212 flow_act.flags |= FLOW_ACT_NO_APPEND; 1213 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 1214 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1215 1216 if (attrs->drop) 1217 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1218 else 1219 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1220 1221 dest[0].ft = tx->ft.status; 1222 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1223 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1224 dest[1].counter_id = mlx5_fc_id(counter); 1225 1226 err = tx_add_kspi_rule(sa_entry, tx, &flow_act, dest, 2); 1227 if (err) { 1228 goto err_add_kspi_rule; 1229 } 1230 1231 err = tx_add_reqid_ip_rules(sa_entry, tx, &flow_act, dest, 2); 1232 if (err) { 1233 goto err_add_reqid_ip_rule; 1234 } 1235 1236 ipsec_rule->fc = counter; 1237 ipsec_rule->pkt_reformat = flow_act.pkt_reformat; 1238 return 0; 1239 1240 err_add_reqid_ip_rule: 1241 mlx5_del_flow_rules(&ipsec_rule->kspi_rule); 1242 err_add_kspi_rule: 1243 mlx5_fc_destroy(mdev, counter); 1244 err_add_cnt: 1245 if (flow_act.pkt_reformat) 1246 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1247 err_pkt_reformat: 1248 tx_ft_put(ipsec); 1249 return err; 1250 } 1251 1252 static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1253 { 1254 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1255 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1256 struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx; 1257 struct mlx5_flow_destination dest[2] = {}; 1258 struct mlx5_flow_act flow_act = {}; 1259 struct mlx5_flow_handle *rule; 1260 struct mlx5_flow_spec *spec; 1261 struct mlx5_flow_table *ft; 1262 int err, dstn = 0; 1263 1264 ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio); 1265 if (IS_ERR(ft)) 1266 return PTR_ERR(ft); 1267 1268 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1269 if (!spec) { 1270 err = -ENOMEM; 1271 goto err_alloc; 1272 } 1273 1274 if (attrs->family == AF_INET) 1275 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1276 else 1277 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1278 1279 setup_fte_no_frags(spec); 1280 setup_fte_upper_proto_match(spec, &attrs->upspec); 1281 1282 switch (attrs->action) { 1283 case IPSEC_POLICY_IPSEC: 1284 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1285 err = setup_modify_header(mdev, attrs->reqid, 1286 IPSEC_DIR_OUTBOUND, &flow_act); 1287 if (err) 1288 goto err_mod_header; 1289 break; 1290 case IPSEC_POLICY_DISCARD: 1291 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 1292 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1293 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1294 dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); 1295 dstn++; 1296 break; 1297 default: 1298 err = -EINVAL; 1299 goto err_mod_header; 1300 } 1301 1302 flow_act.flags |= FLOW_ACT_NO_APPEND; 1303 dest[dstn].ft = tx->ft.sa; 1304 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1305 dstn++; 1306 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1307 if (IS_ERR(rule)) { 1308 err = PTR_ERR(rule); 1309 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1310 goto err_action; 1311 } 1312 1313 kvfree(spec); 1314 pol_entry->ipsec_rule.rule = rule; 1315 pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1316 return 0; 1317 1318 err_action: 1319 if (flow_act.modify_hdr) 1320 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1321 err_mod_header: 1322 kvfree(spec); 1323 err_alloc: 1324 tx_ft_put_policy(pol_entry->ipsec, attrs->prio); 1325 return err; 1326 } 1327 1328 static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1329 { 1330 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1331 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1332 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 1333 struct mlx5_flow_destination dest[2]; 1334 struct mlx5_flow_act flow_act = {}; 1335 struct mlx5_flow_handle *rule; 1336 struct mlx5_flow_spec *spec; 1337 struct mlx5_flow_table *ft; 1338 struct mlx5e_ipsec_rx *rx; 1339 int err, dstn = 0; 1340 1341 rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6; 1342 ft = rx->chains ? ipsec_chains_get_table(rx->chains, attrs->prio) : rx->ft.pol; 1343 if (IS_ERR(ft)) 1344 return PTR_ERR(ft); 1345 1346 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1347 if (!spec) { 1348 err = -ENOMEM; 1349 goto err_alloc; 1350 } 1351 1352 switch (attrs->action) { 1353 case IPSEC_POLICY_IPSEC: 1354 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1355 break; 1356 case IPSEC_POLICY_DISCARD: 1357 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 1358 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1359 dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); 1360 dstn++; 1361 break; 1362 default: 1363 err = -EINVAL; 1364 goto err_action; 1365 } 1366 1367 flow_act.flags |= FLOW_ACT_NO_APPEND; 1368 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1369 dest[dstn].ft = rx->ft.sa; 1370 dstn++; 1371 1372 if (attrs->family == AF_INET) 1373 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1374 else 1375 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1376 1377 setup_fte_no_frags(spec); 1378 setup_fte_upper_proto_match(spec, &attrs->upspec); 1379 if (attrs->vid != VLAN_NONE) 1380 setup_fte_vid(spec, attrs->vid); 1381 else 1382 setup_fte_no_vid(spec); 1383 1384 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1385 if (IS_ERR(rule)) { 1386 err = PTR_ERR(rule); 1387 mlx5_core_err(mdev, 1388 "Failed to add RX IPsec policy rule err=%d\n", err); 1389 goto err_action; 1390 } 1391 pol_entry->ipsec_rule.rule = rule; 1392 1393 /* Add also rule for zero vid */ 1394 if (attrs->vid == VLAN_NONE) { 1395 clear_fte_vid(spec); 1396 setup_fte_vid(spec, 0); 1397 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1398 if (IS_ERR(rule)) { 1399 err = PTR_ERR(rule); 1400 mlx5_core_err(mdev, 1401 "Failed to add RX IPsec policy rule err=%d\n", 1402 err); 1403 goto err_action; 1404 } 1405 pol_entry->ipsec_rule.vid_zero_rule = rule; 1406 } 1407 1408 kvfree(spec); 1409 return 0; 1410 1411 err_action: 1412 if (pol_entry->ipsec_rule.rule != NULL) 1413 mlx5_del_flow_rules(&pol_entry->ipsec_rule.rule); 1414 kvfree(spec); 1415 err_alloc: 1416 if (rx->chains != NULL) 1417 ipsec_chains_put_table(rx->chains, attrs->prio); 1418 return err; 1419 } 1420 1421 static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) 1422 { 1423 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1424 struct mlx5_core_dev *mdev = ipsec->mdev; 1425 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1426 1427 mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); 1428 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1429 kfree(rx_ipv4->fc); 1430 mlx5_fc_destroy(mdev, tx->fc->drop); 1431 mlx5_fc_destroy(mdev, tx->fc->cnt); 1432 kfree(tx->fc); 1433 } 1434 1435 static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) 1436 { 1437 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1438 struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; 1439 struct mlx5_core_dev *mdev = ipsec->mdev; 1440 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1441 struct mlx5e_ipsec_fc *fc; 1442 struct mlx5_fc *counter; 1443 int err; 1444 1445 fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); 1446 if (!fc) 1447 return -ENOMEM; 1448 1449 tx->fc = fc; 1450 counter = mlx5_fc_create(mdev, false); 1451 if (IS_ERR(counter)) { 1452 err = PTR_ERR(counter); 1453 goto err_tx_fc_alloc; 1454 } 1455 1456 fc->cnt = counter; 1457 counter = mlx5_fc_create(mdev, false); 1458 if (IS_ERR(counter)) { 1459 err = PTR_ERR(counter); 1460 goto err_tx_fc_cnt; 1461 } 1462 1463 fc->drop = counter; 1464 1465 fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); 1466 if (!fc) { 1467 err = -ENOMEM; 1468 goto err_tx_fc_drop; 1469 } 1470 1471 /* Both IPv4 and IPv6 point to same flow counters struct. */ 1472 rx_ipv4->fc = fc; 1473 rx_ipv6->fc = fc; 1474 counter = mlx5_fc_create(mdev, false); 1475 if (IS_ERR(counter)) { 1476 err = PTR_ERR(counter); 1477 goto err_rx_fc_alloc; 1478 } 1479 1480 fc->cnt = counter; 1481 counter = mlx5_fc_create(mdev, false); 1482 if (IS_ERR(counter)) { 1483 err = PTR_ERR(counter); 1484 goto err_rx_fc_cnt; 1485 } 1486 1487 fc->drop = counter; 1488 return 0; 1489 1490 err_rx_fc_cnt: 1491 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1492 err_rx_fc_alloc: 1493 kfree(rx_ipv4->fc); 1494 err_tx_fc_drop: 1495 mlx5_fc_destroy(mdev, tx->fc->drop); 1496 err_tx_fc_cnt: 1497 mlx5_fc_destroy(mdev, tx->fc->cnt); 1498 err_tx_fc_alloc: 1499 kfree(tx->fc); 1500 return err; 1501 } 1502 1503 static int ipsec_status_rule(struct mlx5_core_dev *mdev, 1504 struct mlx5e_ipsec_rx *rx, 1505 struct mlx5_flow_destination *dest) 1506 { 1507 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1508 struct mlx5_flow_act flow_act = {}; 1509 struct mlx5_modify_hdr *modify_hdr; 1510 struct mlx5_flow_handle *rule; 1511 struct mlx5_flow_spec *spec; 1512 int err; 1513 1514 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1515 if (!spec) 1516 return -ENOMEM; 1517 1518 /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */ 1519 MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); 1520 MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME); 1521 MLX5_SET(copy_action_in, action, src_offset, 0); 1522 MLX5_SET(copy_action_in, action, length, 7); 1523 MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1524 MLX5_SET(copy_action_in, action, dst_offset, 24); 1525 1526 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 1527 1, action); 1528 1529 if (IS_ERR(modify_hdr)) { 1530 err = PTR_ERR(modify_hdr); 1531 mlx5_core_err(mdev, 1532 "fail to alloc ipsec copy modify_header_id err=%d\n", err); 1533 goto out_spec; 1534 } 1535 1536 /* create fte */ 1537 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 1538 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1539 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1540 flow_act.modify_hdr = modify_hdr; 1541 1542 rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 1543 if (IS_ERR(rule)) { 1544 err = PTR_ERR(rule); 1545 mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); 1546 goto out; 1547 } 1548 1549 kvfree(spec); 1550 rx->status.rule = rule; 1551 rx->status.modify_hdr = modify_hdr; 1552 return 0; 1553 1554 out: 1555 mlx5_modify_header_dealloc(mdev, modify_hdr); 1556 out_spec: 1557 kvfree(spec); 1558 return err; 1559 } 1560 1561 static void ipsec_fs_rx_roce_rules_destroy(struct mlx5e_ipsec_rx_roce *rx_roce) 1562 { 1563 if (!rx_roce->ns_rdma) 1564 return; 1565 1566 mlx5_del_flow_rules(&rx_roce->roce_miss.rule); 1567 mlx5_del_flow_rules(&rx_roce->rule); 1568 mlx5_destroy_flow_group(rx_roce->roce_miss.group); 1569 mlx5_destroy_flow_group(rx_roce->g); 1570 } 1571 1572 static void ipsec_fs_rx_catchall_rules_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) 1573 { 1574 mutex_lock(&rx->ft.mutex); 1575 mlx5_del_flow_rules(&rx->sa.rule); 1576 mlx5_destroy_flow_group(rx->sa.group); 1577 if (rx->chains == NULL) { 1578 mlx5_del_flow_rules(&rx->pol.rule); 1579 mlx5_destroy_flow_group(rx->pol.group); 1580 } 1581 mlx5_del_flow_rules(&rx->status.rule); 1582 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 1583 ipsec_fs_rx_roce_rules_destroy(&rx->roce); 1584 mutex_unlock(&rx->ft.mutex); 1585 } 1586 1587 static void ipsec_fs_rx_roce_table_destroy(struct mlx5e_ipsec_rx_roce *rx_roce) 1588 { 1589 if (!rx_roce->ns_rdma) 1590 return; 1591 1592 mlx5_destroy_flow_table(rx_roce->ft_rdma); 1593 mlx5_destroy_flow_table(rx_roce->ft); 1594 } 1595 1596 static void ipsec_fs_rx_table_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) 1597 { 1598 mutex_lock(&rx->ft.mutex); 1599 if (rx->chains) { 1600 ipsec_chains_destroy(rx->chains); 1601 } else { 1602 mlx5_del_flow_rules(&rx->pol.rule); 1603 mlx5_destroy_flow_table(rx->ft.pol); 1604 } 1605 mlx5_destroy_flow_table(rx->ft.sa); 1606 mlx5_destroy_flow_table(rx->ft.status); 1607 ipsec_fs_rx_roce_table_destroy(&rx->roce); 1608 mutex_unlock(&rx->ft.mutex); 1609 } 1610 1611 static void ipsec_roce_setup_udp_dport(struct mlx5_flow_spec *spec, u16 dport) 1612 { 1613 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1614 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 1615 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 1616 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 1617 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport); 1618 } 1619 1620 static int ipsec_roce_rx_rule_setup(struct mlx5_flow_destination *default_dst, 1621 struct mlx5e_ipsec_rx_roce *roce, struct mlx5_core_dev *mdev) 1622 { 1623 struct mlx5_flow_destination dst = {}; 1624 struct mlx5_flow_act flow_act = {}; 1625 struct mlx5_flow_handle *rule; 1626 struct mlx5_flow_spec *spec; 1627 int err = 0; 1628 1629 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1630 if (!spec) 1631 return -ENOMEM; 1632 1633 ipsec_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT); 1634 1635 //flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;//not needed it is added in command 1636 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 1637 dst.ft = roce->ft_rdma; 1638 1639 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1640 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 1641 if (IS_ERR(rule)) { 1642 err = PTR_ERR(rule); 1643 mlx5_core_err(mdev, "Fail to add RX roce ipsec rule err=%d\n", 1644 err); 1645 goto fail_add_rule; 1646 } 1647 1648 roce->rule = rule; 1649 1650 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, default_dst, 1); 1651 if (IS_ERR(rule)) { 1652 err = PTR_ERR(rule); 1653 mlx5_core_err(mdev, "Fail to add RX roce ipsec miss rule err=%d\n", 1654 err); 1655 goto fail_add_default_rule; 1656 } 1657 1658 roce->roce_miss.rule = rule; 1659 1660 kvfree(spec); 1661 return 0; 1662 1663 fail_add_default_rule: 1664 mlx5_del_flow_rules(&roce->rule); 1665 fail_add_rule: 1666 kvfree(spec); 1667 return err; 1668 } 1669 1670 static int ipsec_roce_rx_rules(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *defdst, 1671 struct mlx5_core_dev *mdev) 1672 { 1673 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1674 struct mlx5_flow_group *g; 1675 void *outer_headers_c; 1676 u32 *in; 1677 int err = 0; 1678 int ix = 0; 1679 u8 *mc; 1680 1681 if (!rx->roce.ns_rdma) 1682 return 0; 1683 1684 in = kvzalloc(inlen, GFP_KERNEL); 1685 if (!in) 1686 return -ENOMEM; 1687 1688 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1689 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 1690 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 1691 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 1692 1693 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1694 MLX5_SET_CFG(in, start_flow_index, ix); 1695 ix += 1; 1696 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1697 g = mlx5_create_flow_group(rx->roce.ft, in); 1698 if (IS_ERR(g)) { 1699 err = PTR_ERR(g); 1700 mlx5_core_err(mdev, "Fail to create ipsec rx roce group at nic err=%d\n", err); 1701 goto fail_group; 1702 } 1703 rx->roce.g = g; 1704 1705 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); 1706 MLX5_SET_CFG(in, start_flow_index, ix); 1707 ix += 1; 1708 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1709 g = mlx5_create_flow_group(rx->roce.ft, in); 1710 if (IS_ERR(g)) { 1711 err = PTR_ERR(g); 1712 mlx5_core_err(mdev, "Fail to create ipsec rx roce miss group at nic err=%d\n", 1713 err); 1714 goto fail_mgroup; 1715 } 1716 rx->roce.roce_miss.group = g; 1717 1718 err = ipsec_roce_rx_rule_setup(defdst, &rx->roce, mdev); 1719 if (err) 1720 goto fail_setup_rule; 1721 1722 kvfree(in); 1723 return 0; 1724 1725 fail_setup_rule: 1726 mlx5_destroy_flow_group(rx->roce.roce_miss.group); 1727 fail_mgroup: 1728 mlx5_destroy_flow_group(rx->roce.g); 1729 fail_group: 1730 kvfree(in); 1731 return err; 1732 } 1733 1734 static int ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv, 1735 struct mlx5e_ipsec_rx *rx, 1736 struct mlx5_flow_destination *defdst) 1737 { 1738 struct mlx5_core_dev *mdev = priv->mdev; 1739 struct mlx5_flow_destination dest[2] = {}; 1740 int err = 0; 1741 1742 mutex_lock(&rx->ft.mutex); 1743 /* IPsec RoCE RX rules */ 1744 err = ipsec_roce_rx_rules(rx, defdst, mdev); 1745 if (err) 1746 goto out; 1747 1748 /* IPsec Rx IP Status table rule */ 1749 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1750 if (rx->roce.ft) 1751 dest[0].ft = rx->roce.ft; 1752 else 1753 dest[0].ft = priv->fts.vlan.t; 1754 1755 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1756 dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); 1757 err = ipsec_status_rule(mdev, rx, dest); 1758 if (err) 1759 goto err_roce_rules_destroy; 1760 1761 if (!rx->chains) { 1762 /* IPsec Rx IP policy default miss rule */ 1763 err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, defdst); 1764 if (err) 1765 goto err_status_rule_destroy; 1766 } 1767 1768 /* FIXME: This is workaround to current design 1769 * which installs SA on firt packet. So we need to forward this 1770 * packet to the stack. It doesn't work with RoCE and eswitch traffic, 1771 */ 1772 err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, defdst); 1773 if (err) 1774 goto err_status_sa_rule_destroy; 1775 1776 mutex_unlock(&rx->ft.mutex); 1777 return 0; 1778 1779 err_status_sa_rule_destroy: 1780 if (!rx->chains) { 1781 mlx5_del_flow_rules(&rx->pol.rule); 1782 mlx5_destroy_flow_group(rx->pol.group); 1783 } 1784 err_status_rule_destroy: 1785 mlx5_del_flow_rules(&rx->status.rule); 1786 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 1787 err_roce_rules_destroy: 1788 ipsec_fs_rx_roce_rules_destroy(&rx->roce); 1789 out: 1790 mutex_unlock(&rx->ft.mutex); 1791 return err; 1792 } 1793 1794 static int ipsec_fs_rx_roce_tables_create(struct mlx5e_ipsec_rx *rx, 1795 int rx_init_level, int rdma_init_level) 1796 { 1797 struct mlx5_flow_table_attr ft_attr = {}; 1798 struct mlx5_flow_table *ft; 1799 int err = 0; 1800 1801 if (!rx->roce.ns_rdma) 1802 return 0; 1803 1804 ft_attr.max_fte = 2; 1805 ft_attr.level = rx_init_level; 1806 ft = mlx5_create_flow_table(rx->ns, &ft_attr); 1807 if (IS_ERR(ft)) { 1808 err = PTR_ERR(ft); 1809 return err; 1810 } 1811 rx->roce.ft = ft; 1812 1813 ft_attr.max_fte = 0; 1814 ft_attr.level = rdma_init_level; 1815 ft = mlx5_create_flow_table(rx->roce.ns_rdma, &ft_attr); 1816 if (IS_ERR(ft)) { 1817 err = PTR_ERR(ft); 1818 goto out; 1819 } 1820 rx->roce.ft_rdma = ft; 1821 1822 return 0; 1823 out: 1824 mlx5_destroy_flow_table(rx->roce.ft); 1825 rx->roce.ft = NULL; 1826 return err; 1827 } 1828 1829 static int ipsec_fs_rx_table_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx, 1830 int rx_init_level, int rdma_init_level) 1831 { 1832 struct mlx5_flow_namespace *ns = rx->ns; 1833 struct mlx5_flow_table *ft; 1834 int err = 0; 1835 1836 mutex_lock(&rx->ft.mutex); 1837 1838 /* IPsec Rx IP SA table create */ 1839 ft = ipsec_rx_ft_create(ns, rx_init_level + 1, 0, 1); 1840 if (IS_ERR(ft)) { 1841 err = PTR_ERR(ft); 1842 goto out; 1843 } 1844 rx->ft.sa = ft; 1845 1846 /* IPsec Rx IP Status table create */ 1847 ft = ipsec_rx_ft_create(ns, rx_init_level + 2, 0, 1); 1848 if (IS_ERR(ft)) { 1849 err = PTR_ERR(ft); 1850 goto err_sa_table_destroy; 1851 } 1852 rx->ft.status = ft; 1853 1854 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 1855 rx->chains = ipsec_chains_create(mdev, rx->ft.sa, 1856 MLX5_FLOW_NAMESPACE_KERNEL, 0, 1857 rx_init_level, &rx->ft.pol); 1858 if (IS_ERR(rx->chains)) { 1859 err = PTR_ERR(rx->chains); 1860 goto err_status_table_destroy; 1861 } 1862 } else { 1863 ft = ipsec_rx_ft_create(ns, rx_init_level, 0, 1); 1864 if (IS_ERR(ft)) { 1865 err = PTR_ERR(ft); 1866 goto err_status_table_destroy; 1867 } 1868 rx->ft.pol = ft; 1869 } 1870 1871 /* IPsec RoCE RX tables create*/ 1872 err = ipsec_fs_rx_roce_tables_create(rx, rx_init_level + 3, 1873 rdma_init_level); 1874 if (err) 1875 goto err_pol_table_destroy; 1876 1877 goto out; 1878 1879 err_pol_table_destroy: 1880 mlx5_destroy_flow_table(rx->ft.pol); 1881 err_status_table_destroy: 1882 mlx5_destroy_flow_table(rx->ft.status); 1883 err_sa_table_destroy: 1884 mlx5_destroy_flow_table(rx->ft.sa); 1885 out: 1886 mutex_unlock(&rx->ft.mutex); 1887 return err; 1888 } 1889 1890 #define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA | MLX5_FT_NIC_TX_RDMA_2_NIC_TX) 1891 1892 static void mlx5e_accel_ipsec_fs_init_roce(struct mlx5e_ipsec *ipsec) 1893 { 1894 struct mlx5_core_dev *mdev = ipsec->mdev; 1895 struct mlx5_flow_namespace *ns; 1896 1897 if ((MLX5_CAP_GEN_2(ipsec->mdev, flow_table_type_2_type) & 1898 NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) { 1899 mlx5_core_dbg(mdev, "Failed to init roce ns, capabilities not supported\n"); 1900 return; 1901 } 1902 1903 ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC); 1904 if (!ns) { 1905 mlx5_core_err(mdev, "Failed to init roce rx ns\n"); 1906 return; 1907 } 1908 1909 ipsec->rx_ipv4->roce.ns_rdma = ns; 1910 ipsec->rx_ipv6->roce.ns_rdma = ns; 1911 1912 ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC); 1913 if (!ns) { 1914 ipsec->rx_ipv4->roce.ns_rdma = NULL; 1915 ipsec->rx_ipv6->roce.ns_rdma = NULL; 1916 mlx5_core_err(mdev, "Failed to init roce tx ns\n"); 1917 return; 1918 } 1919 1920 ipsec->tx->roce.ns = ns; 1921 } 1922 1923 int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1924 { 1925 if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) 1926 return tx_add_rule(sa_entry); 1927 1928 return rx_add_rule(sa_entry); 1929 } 1930 1931 void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1932 { 1933 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1934 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1935 1936 mlx5_del_flow_rules(&ipsec_rule->rule); 1937 mlx5_del_flow_rules(&ipsec_rule->kspi_rule); 1938 if (ipsec_rule->vid_zero_rule != NULL) 1939 mlx5_del_flow_rules(&ipsec_rule->vid_zero_rule); 1940 if (ipsec_rule->reqid_rule != NULL) 1941 mlx5_del_flow_rules(&ipsec_rule->reqid_rule); 1942 mlx5_fc_destroy(mdev, ipsec_rule->fc); 1943 mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); 1944 if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) { 1945 tx_ft_put(sa_entry->ipsec); 1946 return; 1947 } 1948 1949 if (ipsec_rule->modify_hdr != NULL) 1950 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1951 } 1952 1953 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1954 { 1955 if (pol_entry->attrs.dir == IPSEC_DIR_OUTBOUND) 1956 return tx_add_policy(pol_entry); 1957 1958 return rx_add_policy(pol_entry); 1959 } 1960 1961 void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1962 { 1963 struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; 1964 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1965 1966 mlx5_del_flow_rules(&ipsec_rule->rule); 1967 if (ipsec_rule->vid_zero_rule != NULL) 1968 mlx5_del_flow_rules(&ipsec_rule->vid_zero_rule); 1969 1970 if (pol_entry->attrs.dir == IPSEC_DIR_INBOUND) { 1971 struct mlx5e_ipsec_rx *rx; 1972 1973 rx = (pol_entry->attrs.family == AF_INET) 1974 ? pol_entry->ipsec->rx_ipv4 1975 : pol_entry->ipsec->rx_ipv6; 1976 if (rx->chains) 1977 ipsec_chains_put_table(rx->chains, 1978 pol_entry->attrs.prio); 1979 return; 1980 } 1981 1982 if (ipsec_rule->modify_hdr) 1983 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1984 1985 tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio); 1986 } 1987 1988 void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv) 1989 { 1990 /* Check if IPsec supported */ 1991 if (!priv->ipsec) 1992 return; 1993 1994 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv4); 1995 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6); 1996 } 1997 1998 int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv) 1999 { 2000 struct mlx5e_ipsec *ipsec = priv->ipsec; 2001 struct mlx5_flow_destination dest = {}; 2002 int err = 0; 2003 2004 /* Check if IPsec supported */ 2005 if (!ipsec) 2006 return 0; 2007 2008 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2009 dest.ft = priv->fts.vlan.t; 2010 err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv6, &dest); 2011 if (err) 2012 goto out; 2013 2014 err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv4, &dest); 2015 if (err) 2016 ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6); 2017 out: 2018 return err; 2019 } 2020 2021 void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv) 2022 { 2023 struct mlx5_core_dev *mdev = priv->mdev; 2024 struct mlx5e_ipsec *ipsec = priv->ipsec; 2025 2026 /* Check if IPsec supported */ 2027 if (!ipsec) 2028 return; 2029 2030 ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv6); 2031 ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv4); 2032 } 2033 2034 int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv) 2035 { 2036 struct mlx5e_ipsec *ipsec = priv->ipsec; 2037 int err = 0; 2038 2039 /* Check if IPsec supported */ 2040 if (!ipsec) 2041 return 0; 2042 2043 err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv4, 0, 0); 2044 if (err) 2045 goto out; 2046 2047 err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv6, 4, 1); 2048 if (err) { 2049 ipsec_fs_rx_table_destroy(priv->mdev, ipsec->rx_ipv4); 2050 goto out; 2051 } 2052 2053 priv->fts.ipsec_ft = priv->ipsec->rx_ipv4->ft.pol; 2054 out: 2055 return err; 2056 } 2057 2058 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 2059 { 2060 WARN_ON(ipsec->tx->ft.refcnt); 2061 mutex_destroy(&ipsec->rx_ipv6->ft.mutex); 2062 mutex_destroy(&ipsec->rx_ipv4->ft.mutex); 2063 mutex_destroy(&ipsec->tx->ft.mutex); 2064 ipsec_fs_destroy_counters(ipsec); 2065 kfree(ipsec->rx_ipv6); 2066 kfree(ipsec->rx_ipv4); 2067 kfree(ipsec->tx); 2068 } 2069 2070 int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) 2071 { 2072 struct mlx5_flow_namespace *tns, *rns; 2073 int err = -ENOMEM; 2074 2075 tns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 2076 if (!tns) 2077 return -EOPNOTSUPP; 2078 2079 rns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_KERNEL); 2080 if (!rns) 2081 return -EOPNOTSUPP; 2082 2083 ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); 2084 if (!ipsec->tx) 2085 return -ENOMEM; 2086 2087 ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); 2088 if (!ipsec->rx_ipv4) 2089 goto err_tx; 2090 2091 ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); 2092 if (!ipsec->rx_ipv6) 2093 goto err_rx_ipv4; 2094 2095 err = ipsec_fs_init_counters(ipsec); 2096 if (err) 2097 goto err_rx_ipv6; 2098 2099 ipsec->tx->ns = tns; 2100 mutex_init(&ipsec->tx->ft.mutex); 2101 ipsec->rx_ipv4->ns = rns; 2102 ipsec->rx_ipv6->ns = rns; 2103 mutex_init(&ipsec->rx_ipv4->ft.mutex); 2104 mutex_init(&ipsec->rx_ipv6->ft.mutex); 2105 2106 mlx5e_accel_ipsec_fs_init_roce(ipsec); 2107 2108 return 0; 2109 2110 err_rx_ipv6: 2111 kfree(ipsec->rx_ipv6); 2112 err_rx_ipv4: 2113 kfree(ipsec->rx_ipv4); 2114 err_tx: 2115 kfree(ipsec->tx); 2116 return err; 2117 } 2118 2119 void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) 2120 { 2121 struct mlx5e_ipsec_sa_entry sa_entry_shadow = {}; 2122 int err; 2123 2124 memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry)); 2125 memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule)); 2126 2127 err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow); 2128 if (err) 2129 return; 2130 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 2131 memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry)); 2132 } 2133