1 /*- 2 * Copyright (c) 2023 NVIDIA corporation & affiliates. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include "opt_ipsec.h" 28 29 #include <sys/types.h> 30 #include <netinet/in.h> 31 #include <sys/socket.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <net/if.h> 35 #include <net/if_var.h> 36 #include <net/pfkeyv2.h> 37 #include <netipsec/key_var.h> 38 #include <netipsec/keydb.h> 39 #include <netipsec/ipsec.h> 40 #include <netipsec/xform.h> 41 #include <netipsec/ipsec_offload.h> 42 #include <dev/mlx5/fs.h> 43 #include <dev/mlx5/mlx5_en/en.h> 44 #include <dev/mlx5/mlx5_accel/ipsec.h> 45 46 #define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000) 47 48 static void mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, 49 void *priv); 50 static int mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv); 51 52 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x) 53 { 54 return (struct mlx5e_ipsec_sa_entry *)x; 55 } 56 57 static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x) 58 { 59 return (struct mlx5e_ipsec_pol_entry *)x; 60 } 61 62 static void 63 mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry *sa_entry, 64 u64 *packets, u64 *bytes) 65 { 66 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 67 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 68 69 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 70 } 71 72 static struct mlx5e_ipsec_sa_entry * 73 mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir *pb, 74 struct mlx5e_ipsec_sa_entry *sa_entry) 75 { 76 return (pb->priv_in == sa_entry ? pb->priv_out : pb->priv_in); 77 } 78 79 static void 80 mlx5e_ipsec_handle_counters(struct work_struct *_work) 81 { 82 struct mlx5e_ipsec_dwork *dwork = 83 container_of(_work, struct mlx5e_ipsec_dwork, dwork.work); 84 struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry; 85 struct mlx5e_ipsec_sa_entry *other_sa_entry; 86 u64 bytes, bytes1, packets1, packets; 87 88 if (sa_entry->attrs.drop) 89 return; 90 other_sa_entry = mlx5e_ipsec_other_sa_entry(dwork->pb, sa_entry); 91 if (other_sa_entry == NULL || other_sa_entry->attrs.drop) 92 return; 93 94 mlx5e_ipsec_handle_counters_onedir(sa_entry, &packets, &bytes); 95 mlx5e_ipsec_handle_counters_onedir(other_sa_entry, &packets1, &bytes1); 96 packets += packets1; 97 bytes += bytes1; 98 99 #ifdef IPSEC_OFFLOAD 100 ipsec_accel_drv_sa_lifetime_update( 101 sa_entry->savp, sa_entry->ifpo, sa_entry->kspi, bytes, packets); 102 #endif 103 104 queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, 105 MLX5_IPSEC_RESCHED); 106 } 107 108 static int 109 mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry, 110 struct mlx5e_ipsec_priv_bothdir *pb) 111 { 112 struct mlx5e_ipsec_dwork *dwork; 113 114 dwork = kzalloc(sizeof(*dwork), GFP_KERNEL); 115 if (!dwork) 116 return (ENOMEM); 117 118 dwork->sa_entry = sa_entry; 119 dwork->pb = pb; 120 INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters); 121 sa_entry->dwork = dwork; 122 return 0; 123 } 124 125 static int mlx5_xform_ah_authsize(const struct auth_hash *esph) 126 { 127 int alen; 128 129 if (esph == NULL) 130 return 0; 131 132 switch (esph->type) { 133 case CRYPTO_SHA2_256_HMAC: 134 case CRYPTO_SHA2_384_HMAC: 135 case CRYPTO_SHA2_512_HMAC: 136 alen = esph->hashsize / 2; /* RFC4868 2.3 */ 137 break; 138 139 case CRYPTO_POLY1305: 140 case CRYPTO_AES_NIST_GMAC: 141 alen = esph->hashsize; 142 break; 143 144 default: 145 alen = AH_HMAC_HASHLEN; 146 break; 147 } 148 149 return alen; 150 } 151 152 void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 153 struct mlx5_accel_esp_xfrm_attrs *attrs, 154 u8 dir) 155 { 156 struct secasvar *savp = sa_entry->savp; 157 const struct auth_hash *esph = savp->tdb_authalgxform; 158 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 159 struct secasindex *saidx = &savp->sah->saidx; 160 struct seckey *key_encap = savp->key_enc; 161 int key_len; 162 163 memset(attrs, 0, sizeof(*attrs)); 164 165 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 166 key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 167 168 memcpy(aes_gcm->aes_key, key_encap->key_data, key_len); 169 aes_gcm->key_len = key_len; 170 171 /* salt and seq_iv */ 172 aes_gcm->seq_iv = 0; 173 memcpy(&aes_gcm->salt, key_encap->key_data + key_len, 174 sizeof(aes_gcm->salt)); 175 176 switch (savp->alg_enc) { 177 case SADB_X_EALG_AESGCM8: 178 attrs->authsize = 8 / 4; /* in dwords */ 179 break; 180 case SADB_X_EALG_AESGCM12: 181 attrs->authsize = 12 / 4; /* in dwords */ 182 break; 183 case SADB_X_EALG_AESGCM16: 184 attrs->authsize = 16 / 4; /* in dwords */ 185 break; 186 default: break; 187 } 188 189 /* iv len */ 190 aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense 191 192 attrs->dir = dir; 193 /* spi - host order */ 194 attrs->spi = ntohl(savp->spi); 195 attrs->family = saidx->dst.sa.sa_family; 196 attrs->reqid = saidx->reqid; 197 198 if (saidx->src.sa.sa_family == AF_INET) { 199 attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr; 200 attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr; 201 } else { 202 memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16); 203 memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16); 204 } 205 206 if (savp->natt) { 207 attrs->encap = true; 208 attrs->sport = savp->natt->sport; 209 attrs->dport = savp->natt->dport; 210 } 211 212 if (savp->flags & SADB_X_SAFLAGS_ESN) { 213 /* We support replay window with ESN only */ 214 attrs->replay_esn.trigger = true; 215 if (sa_entry->esn_state.esn_msb) 216 attrs->replay_esn.esn = sa_entry->esn_state.esn; 217 else 218 /* According to RFC4303, section "3.3.3. Sequence Number Generation", 219 * the first packet sent using a given SA will contain a sequence 220 * number of 1. 221 */ 222 attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1); 223 attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb; 224 attrs->replay_esn.overlap = sa_entry->esn_state.overlap; 225 226 if (savp->replay) { 227 switch (savp->replay->wsize) { 228 case 4: 229 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT; 230 break; 231 case 8: 232 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT; 233 break; 234 case 16: 235 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT; 236 break; 237 case 32: 238 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT; 239 break; 240 default: 241 /* Do nothing */ 242 break; 243 } 244 } 245 } 246 } 247 248 static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, 249 struct secasvar *savp) 250 { 251 struct secasindex *saidx = &savp->sah->saidx; 252 struct seckey *key_encp = savp->key_enc; 253 int keylen; 254 255 if (!(mlx5_ipsec_device_caps(mdev) & 256 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 257 mlx5_core_err(mdev, "FULL offload is not supported\n"); 258 return (EINVAL); 259 } 260 if (savp->alg_enc == SADB_EALG_NONE) { 261 mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n"); 262 return (EINVAL); 263 } 264 if (savp->alg_enc != SADB_X_EALG_AESGCM16) { 265 mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n"); 266 return (EINVAL); 267 } 268 if (savp->tdb_compalgxform) { 269 mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n"); 270 return (EINVAL); 271 } 272 if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) { 273 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n"); 274 return (EINVAL); 275 } 276 if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) || 277 (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) { 278 mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n"); 279 return (EINVAL); 280 } 281 if (saidx->proto != IPPROTO_ESP) { 282 mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n"); 283 return (EINVAL); 284 } 285 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 286 keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 287 if (keylen != 128/8 && keylen != 256 / 8) { 288 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 289 return (EINVAL); 290 } 291 292 if (saidx->mode != IPSEC_MODE_TRANSPORT) { 293 mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n"); 294 return (EINVAL); 295 } 296 297 if (savp->natt) { 298 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) { 299 mlx5_core_err(mdev, "Encapsulation is not supported\n"); 300 return (EINVAL); 301 } 302 } 303 304 if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 && 305 savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) { 306 mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize); 307 return (EINVAL); 308 } 309 310 if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) { 311 if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) { 312 mlx5_core_err(mdev, "ESN is not supported\n"); 313 return (EINVAL); 314 } 315 } else if (savp->replay != NULL && savp->replay->wsize != 0) { 316 mlx5_core_warn(mdev, 317 "non-ESN but replay-protect SA offload is not supported\n"); 318 return (EINVAL); 319 } 320 return 0; 321 } 322 323 static int 324 mlx5e_if_sa_newkey_onedir(struct ifnet *ifp, void *sav, int dir, u_int drv_spi, 325 struct mlx5e_ipsec_sa_entry **privp, struct mlx5e_ipsec_priv_bothdir *pb, 326 struct ifnet *ifpo) 327 { 328 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 329 struct mlx5e_priv *priv = if_getsoftc(ifp); 330 struct mlx5_core_dev *mdev = priv->mdev; 331 struct mlx5e_ipsec *ipsec = priv->ipsec; 332 u16 vid = VLAN_NONE; 333 int err; 334 335 if (priv->gone != 0 || ipsec == NULL) 336 return (EOPNOTSUPP); 337 338 if (if_gettype(ifpo) == IFT_L2VLAN) 339 VLAN_TAG(ifpo, &vid); 340 341 err = mlx5e_xfrm_validate_state(mdev, sav); 342 if (err) 343 return err; 344 345 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 346 if (sa_entry == NULL) 347 return (ENOMEM); 348 349 sa_entry->kspi = drv_spi; 350 sa_entry->savp = sav; 351 sa_entry->ifp = ifp; 352 sa_entry->ifpo = ifpo; 353 sa_entry->ipsec = ipsec; 354 sa_entry->vid = vid; 355 356 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir); 357 358 err = mlx5e_ipsec_create_dwork(sa_entry, pb); 359 if (err) 360 goto err_xfrm; 361 362 /* create hw context */ 363 err = mlx5_ipsec_create_sa_ctx(sa_entry); 364 if (err) 365 goto err_sa_ctx; 366 367 err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); 368 if (err) 369 goto err_fs; 370 371 *privp = sa_entry; 372 if (sa_entry->dwork) 373 queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED); 374 375 err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL); 376 if (err) 377 goto err_xa; 378 379 return 0; 380 381 err_xa: 382 if (sa_entry->dwork) 383 cancel_delayed_work_sync(&sa_entry->dwork->dwork); 384 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 385 err_fs: 386 mlx5_ipsec_free_sa_ctx(sa_entry); 387 err_sa_ctx: 388 kfree(sa_entry->dwork); 389 sa_entry->dwork = NULL; 390 err_xfrm: 391 kfree(sa_entry); 392 mlx5_en_err(ifp, "Device failed to offload this state"); 393 return err; 394 } 395 396 #define GET_TRUNK_IF(vifp, ifp, ept) \ 397 if (if_gettype(vifp) == IFT_L2VLAN) { \ 398 NET_EPOCH_ENTER(ept); \ 399 ifp = VLAN_TRUNKDEV(vifp); \ 400 NET_EPOCH_EXIT(ept); \ 401 } else { \ 402 ifp = vifp; \ 403 } 404 405 static int 406 mlx5e_if_sa_newkey(struct ifnet *ifpo, void *sav, u_int dev_spi, void **privp) 407 { 408 struct mlx5e_ipsec_priv_bothdir *pb; 409 struct epoch_tracker et; 410 struct ifnet *ifp; 411 int error; 412 413 GET_TRUNK_IF(ifpo, ifp, et); 414 415 pb = malloc(sizeof(struct mlx5e_ipsec_priv_bothdir), M_DEVBUF, 416 M_WAITOK | M_ZERO); 417 error = mlx5e_if_sa_newkey_onedir( 418 ifp, sav, IPSEC_DIR_INBOUND, dev_spi, &pb->priv_in, pb, ifpo); 419 if (error != 0) { 420 free(pb, M_DEVBUF); 421 return (error); 422 } 423 error = mlx5e_if_sa_newkey_onedir( 424 ifp, sav, IPSEC_DIR_OUTBOUND, dev_spi, &pb->priv_out, pb, ifpo); 425 if (error == 0) { 426 *privp = pb; 427 } else { 428 if (pb->priv_in->dwork != NULL) 429 cancel_delayed_work_sync(&pb->priv_in->dwork->dwork); 430 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb->priv_in); 431 free(pb, M_DEVBUF); 432 } 433 return (error); 434 } 435 436 static void 437 mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, void *priv) 438 { 439 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 440 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 441 struct mlx5e_ipsec_sa_entry *old; 442 443 old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id); 444 WARN_ON(old != sa_entry); 445 446 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 447 mlx5_ipsec_free_sa_ctx(sa_entry); 448 kfree(sa_entry->dwork); 449 kfree(sa_entry); 450 } 451 452 static int 453 mlx5e_if_sa_deinstall(struct ifnet *ifpo, u_int dev_spi, void *priv) 454 { 455 struct mlx5e_ipsec_priv_bothdir pb, *pbp; 456 struct epoch_tracker et; 457 struct ifnet *ifp; 458 459 GET_TRUNK_IF(ifpo, ifp, et); 460 461 pbp = priv; 462 pb = *(struct mlx5e_ipsec_priv_bothdir *)priv; 463 pbp->priv_in = pbp->priv_out = NULL; 464 465 if (pb.priv_in->dwork != NULL) 466 cancel_delayed_work_sync(&pb.priv_in->dwork->dwork); 467 if (pb.priv_out->dwork != NULL) 468 cancel_delayed_work_sync(&pb.priv_out->dwork->dwork); 469 470 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_in); 471 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_out); 472 free(pbp, M_DEVBUF); 473 return (0); 474 } 475 476 static void 477 mlx5e_if_sa_cnt_one(struct ifnet *ifp, void *sa, uint32_t drv_spi, 478 void *priv, u64 *bytes, u64 *packets) 479 { 480 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 481 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 482 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 483 484 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 485 } 486 487 static int 488 mlx5e_if_sa_cnt(struct ifnet *ifpo, void *sa, uint32_t drv_spi, void *priv, 489 struct seclifetime *lt) 490 { 491 struct mlx5e_ipsec_priv_bothdir *pb; 492 u64 packets_in, packets_out; 493 u64 bytes_in, bytes_out; 494 struct epoch_tracker et; 495 struct ifnet *ifp; 496 497 GET_TRUNK_IF(ifpo, ifp, et); 498 499 pb = priv; 500 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_in, 501 &bytes_in, &packets_in); 502 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_out, 503 &bytes_out, &packets_out); 504 /* TODO: remove this casting once Kostia changes allocation type to be u64 */ 505 lt->bytes = bytes_in + bytes_out; 506 lt->allocations = (uint32_t)(packets_in + packets_out); 507 return (0); 508 } 509 510 static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, 511 struct secpolicy *sp, struct inpcb *inp) 512 { 513 struct secpolicyindex *spidx = &sp->spidx; 514 515 if (!(mlx5_ipsec_device_caps(mdev) & 516 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 517 mlx5_core_err(mdev, "FULL offload is not supported\n"); 518 return (EINVAL); 519 } 520 521 if (sp->tcount > 1) { 522 mlx5_core_err(mdev, "Can offload exactly one template, " 523 "not %d\n", sp->tcount); 524 return (EINVAL); 525 } 526 527 if (sp->policy == IPSEC_POLICY_BYPASS && 528 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) { 529 mlx5_core_err(mdev, "Device does not support policy priority\n"); 530 return (EINVAL); 531 } 532 533 if (sp->tcount > 0 && inp != NULL) { 534 mlx5_core_err(mdev, "Not valid input data\n"); 535 return (EINVAL); 536 } 537 538 if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) { 539 mlx5_core_err(mdev, "Wrong policy direction\n"); 540 return (EINVAL); 541 } 542 543 if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) { 544 mlx5_core_err(mdev, "Device supports transport mode only"); 545 return (EINVAL); 546 } 547 548 if (sp->policy != IPSEC_POLICY_DISCARD && 549 sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) { 550 mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n"); 551 return (EINVAL); 552 } 553 554 if (sp->policy == IPSEC_POLICY_BYPASS && !inp) { 555 mlx5_core_err(mdev, "Missing port information for IKE bypass\n"); 556 return (EINVAL); 557 } 558 559 if (inp != NULL) { 560 INP_RLOCK(inp); 561 if (inp->inp_socket == NULL || inp->inp_socket->so_proto-> 562 pr_protocol != IPPROTO_UDP) { 563 mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n", 564 inp->inp_socket == NULL ? -1 : 565 inp->inp_socket->so_proto->pr_protocol); 566 INP_RUNLOCK(inp); 567 return (EINVAL); 568 } 569 INP_RUNLOCK(inp); 570 } 571 572 /* TODO fill relevant bits */ 573 return 0; 574 } 575 576 static void 577 mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, 578 struct mlx5_accel_pol_xfrm_attrs *attrs, struct inpcb *inp, u16 vid) 579 { 580 struct secpolicy *sp = pol_entry->sp; 581 struct secpolicyindex *spidx = &sp->spidx; 582 583 memset(attrs, 0, sizeof(*attrs)); 584 585 if (!inp) { 586 if (spidx->src.sa.sa_family == AF_INET) { 587 attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr; 588 attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr; 589 } else if (spidx->src.sa.sa_family == AF_INET6) { 590 memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16); 591 memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16); 592 } else { 593 KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family)); 594 } 595 attrs->family = spidx->src.sa.sa_family; 596 attrs->prio = 0; 597 attrs->action = sp->policy; 598 attrs->reqid = sp->req[0]->saidx.reqid; 599 } else { 600 INP_RLOCK(inp); 601 if ((inp->inp_vflag & INP_IPV4) != 0) { 602 attrs->saddr.a4 = inp->inp_laddr.s_addr; 603 attrs->daddr.a4 = inp->inp_faddr.s_addr; 604 attrs->family = AF_INET; 605 } else if ((inp->inp_vflag & INP_IPV6) != 0) { 606 memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16); 607 memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16); 608 attrs->family = AF_INET6; 609 } else { 610 KASSERT(0, ("unsupported family %d", inp->inp_vflag)); 611 } 612 attrs->upspec.dport = inp->inp_fport; 613 attrs->upspec.sport = inp->inp_lport; 614 attrs->upspec.proto = inp->inp_ip_p; 615 INP_RUNLOCK(inp); 616 617 /* Give highest priority for PCB policies */ 618 attrs->prio = 1; 619 attrs->action = IPSEC_POLICY_IPSEC; 620 } 621 attrs->dir = spidx->dir; 622 attrs->vid = vid; 623 } 624 625 static int 626 mlx5e_if_spd_install(struct ifnet *ifpo, void *sp, void *inp1, void **ifdatap) 627 { 628 struct mlx5e_ipsec_pol_entry *pol_entry; 629 struct mlx5e_priv *priv; 630 struct epoch_tracker et; 631 u16 vid = VLAN_NONE; 632 struct ifnet *ifp; 633 int err; 634 635 GET_TRUNK_IF(ifpo, ifp, et); 636 if (if_gettype(ifpo) == IFT_L2VLAN) 637 VLAN_TAG(ifpo, &vid); 638 priv = if_getsoftc(ifp); 639 if (priv->gone || !priv->ipsec) 640 return (EOPNOTSUPP); 641 642 err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1); 643 if (err) 644 return err; 645 646 pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); 647 if (!pol_entry) 648 return (ENOMEM); 649 650 pol_entry->sp = sp; 651 pol_entry->ipsec = priv->ipsec; 652 653 mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, 654 inp1, vid); 655 err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); 656 if (err) 657 goto err_pol; 658 *ifdatap = pol_entry; 659 660 return 0; 661 662 err_pol: 663 kfree(pol_entry); 664 mlx5_en_err(ifp, "Device failed to offload this policy"); 665 return err; 666 } 667 668 static int 669 mlx5e_if_spd_deinstall(struct ifnet *ifpo, void *sp, void *ifdata) 670 { 671 struct mlx5e_ipsec_pol_entry *pol_entry; 672 673 pol_entry = to_ipsec_pol_entry(ifdata); 674 mlx5e_accel_ipsec_fs_del_pol(pol_entry); 675 kfree(pol_entry); 676 return 0; 677 } 678 679 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 680 { 681 struct mlx5e_ipsec *pipsec = priv->ipsec; 682 if (!pipsec) 683 return; 684 685 mlx5e_accel_ipsec_fs_cleanup(pipsec); 686 destroy_workqueue(pipsec->wq); 687 mlx5e_ipsec_aso_cleanup(pipsec); 688 kfree(pipsec); 689 priv->ipsec = NULL; 690 } 691 692 static int 693 mlx5e_if_ipsec_hwassist(if_t ifneto, void *sav __unused, 694 uint32_t drv_spi __unused, void *priv __unused) 695 { 696 if_t ifnet; 697 698 if (if_gettype(ifneto) == IFT_L2VLAN) { 699 ifnet = VLAN_TRUNKDEV(ifneto); 700 } else { 701 ifnet = ifneto; 702 } 703 704 return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP | 705 CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP)); 706 } 707 708 static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = { 709 .if_sa_newkey = mlx5e_if_sa_newkey, 710 .if_sa_deinstall = mlx5e_if_sa_deinstall, 711 .if_spdadd = mlx5e_if_spd_install, 712 .if_spddel = mlx5e_if_spd_deinstall, 713 .if_sa_cnt = mlx5e_if_sa_cnt, 714 .if_hwassist = mlx5e_if_ipsec_hwassist, 715 }; 716 717 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 718 { 719 struct mlx5_core_dev *mdev = priv->mdev; 720 struct mlx5e_ipsec *pipsec; 721 if_t ifp = priv->ifp; 722 int ret; 723 724 mlx5_core_info(mdev, "ipsec " 725 "offload %d log_max_dek %d gen_obj_types %d " 726 "ipsec_encrypt %d ipsec_decrypt %d " 727 "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d " 728 "ipsec_full_offload %d " 729 "reformat_add_esp_trasport %d reformat_del_esp_trasport %d " 730 "decap %d " 731 "ignore_flow_level_tx %d ignore_flow_level_rx %d " 732 "reformat_natt_tx %d reformat_natt_rx %d " 733 "ipsec_esn %d\n", 734 MLX5_CAP_GEN(mdev, ipsec_offload) != 0, 735 MLX5_CAP_GEN(mdev, log_max_dek) != 0, 736 (MLX5_CAP_GEN_64(mdev, general_obj_types) & 737 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0, 738 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0, 739 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0, 740 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0, 741 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0, 742 MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0, 743 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0, 744 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0, 745 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0, 746 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0, 747 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0, 748 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, 749 reformat_add_esp_transport_over_udp) != 0, 750 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 751 reformat_del_esp_transport_over_udp) != 0, 752 MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0); 753 754 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 755 mlx5_core_dbg(mdev, "Not an IPSec offload device\n"); 756 return 0; 757 } 758 759 xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC); 760 761 pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL); 762 if (pipsec == NULL) 763 return (ENOMEM); 764 765 pipsec->mdev = mdev; 766 pipsec->pdn = priv->pdn; 767 pipsec->mkey = priv->mr.key; 768 769 ret = mlx5e_ipsec_aso_init(pipsec); 770 if (ret) 771 goto err_ipsec_aso; 772 773 pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0); 774 if (pipsec->wq == NULL) { 775 ret = ENOMEM; 776 goto err_ipsec_wq; 777 } 778 779 ret = mlx5e_accel_ipsec_fs_init(pipsec); 780 if (ret) 781 goto err_ipsec_alloc; 782 783 if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs); 784 priv->ipsec = pipsec; 785 mlx5_core_dbg(mdev, "IPSec attached to netdevice\n"); 786 return 0; 787 788 err_ipsec_alloc: 789 destroy_workqueue(pipsec->wq); 790 err_ipsec_wq: 791 mlx5e_ipsec_aso_cleanup(pipsec); 792 err_ipsec_aso: 793 kfree(pipsec); 794 mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret); 795 return ret; 796 } 797