1 /*- 2 * Copyright (c) 2023 NVIDIA corporation & affiliates. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include "opt_ipsec.h" 28 29 #include <sys/types.h> 30 #include <netinet/in.h> 31 #include <sys/socket.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <net/if.h> 35 #include <net/if_var.h> 36 #include <net/pfkeyv2.h> 37 #include <netipsec/key.h> 38 #include <netipsec/key_var.h> 39 #include <netipsec/keydb.h> 40 #include <netipsec/ipsec.h> 41 #include <netipsec/xform.h> 42 #include <netipsec/ipsec_offload.h> 43 #include <dev/mlx5/fs.h> 44 #include <dev/mlx5/mlx5_en/en.h> 45 #include <dev/mlx5/mlx5_accel/ipsec.h> 46 47 #define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000) 48 49 static void mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, 50 void *priv); 51 static int mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv); 52 53 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x) 54 { 55 return (struct mlx5e_ipsec_sa_entry *)x; 56 } 57 58 static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x) 59 { 60 return (struct mlx5e_ipsec_pol_entry *)x; 61 } 62 63 static void 64 mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry *sa_entry, 65 u64 *packets, u64 *bytes) 66 { 67 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 68 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 69 70 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 71 } 72 73 static struct mlx5e_ipsec_sa_entry * 74 mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir *pb, 75 struct mlx5e_ipsec_sa_entry *sa_entry) 76 { 77 return (pb->priv_in == sa_entry ? pb->priv_out : pb->priv_in); 78 } 79 80 static void 81 mlx5e_ipsec_handle_counters(struct work_struct *_work) 82 { 83 struct mlx5e_ipsec_dwork *dwork = 84 container_of(_work, struct mlx5e_ipsec_dwork, dwork.work); 85 struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry; 86 struct mlx5e_ipsec_sa_entry *other_sa_entry; 87 u64 bytes, bytes1, packets1, packets; 88 89 if (sa_entry->attrs.drop) 90 return; 91 other_sa_entry = mlx5e_ipsec_other_sa_entry(dwork->pb, sa_entry); 92 if (other_sa_entry == NULL || other_sa_entry->attrs.drop) 93 return; 94 95 mlx5e_ipsec_handle_counters_onedir(sa_entry, &packets, &bytes); 96 mlx5e_ipsec_handle_counters_onedir(other_sa_entry, &packets1, &bytes1); 97 packets += packets1; 98 bytes += bytes1; 99 100 #ifdef IPSEC_OFFLOAD 101 ipsec_accel_drv_sa_lifetime_update( 102 sa_entry->savp, sa_entry->ifpo, sa_entry->kspi, bytes, packets); 103 #endif 104 105 queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, 106 MLX5_IPSEC_RESCHED); 107 } 108 109 static int 110 mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry, 111 struct mlx5e_ipsec_priv_bothdir *pb) 112 { 113 struct mlx5e_ipsec_dwork *dwork; 114 115 dwork = kzalloc(sizeof(*dwork), GFP_KERNEL); 116 if (!dwork) 117 return (ENOMEM); 118 119 dwork->sa_entry = sa_entry; 120 dwork->pb = pb; 121 INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters); 122 sa_entry->dwork = dwork; 123 return 0; 124 } 125 126 static int mlx5_xform_ah_authsize(const struct auth_hash *esph) 127 { 128 int alen; 129 130 if (esph == NULL) 131 return 0; 132 133 switch (esph->type) { 134 case CRYPTO_SHA2_256_HMAC: 135 case CRYPTO_SHA2_384_HMAC: 136 case CRYPTO_SHA2_512_HMAC: 137 alen = esph->hashsize / 2; /* RFC4868 2.3 */ 138 break; 139 140 case CRYPTO_POLY1305: 141 case CRYPTO_AES_NIST_GMAC: 142 alen = esph->hashsize; 143 break; 144 145 default: 146 alen = AH_HMAC_HASHLEN; 147 break; 148 } 149 150 return alen; 151 } 152 153 void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 154 struct mlx5_accel_esp_xfrm_attrs *attrs, 155 u8 dir) 156 { 157 struct secasvar *savp = sa_entry->savp; 158 const struct auth_hash *esph = savp->tdb_authalgxform; 159 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 160 struct secasindex *saidx = &savp->sah->saidx; 161 struct seckey *key_encap = savp->key_enc; 162 int key_len; 163 164 memset(attrs, 0, sizeof(*attrs)); 165 166 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 167 key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 168 169 memcpy(aes_gcm->aes_key, key_encap->key_data, key_len); 170 aes_gcm->key_len = key_len; 171 172 /* salt and seq_iv */ 173 aes_gcm->seq_iv = 0; 174 memcpy(&aes_gcm->salt, key_encap->key_data + key_len, 175 sizeof(aes_gcm->salt)); 176 177 switch (savp->alg_enc) { 178 case SADB_X_EALG_AESGCM8: 179 attrs->authsize = 8 / 4; /* in dwords */ 180 break; 181 case SADB_X_EALG_AESGCM12: 182 attrs->authsize = 12 / 4; /* in dwords */ 183 break; 184 case SADB_X_EALG_AESGCM16: 185 attrs->authsize = 16 / 4; /* in dwords */ 186 break; 187 default: break; 188 } 189 190 /* iv len */ 191 aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense 192 193 attrs->dir = dir; 194 /* spi - host order */ 195 attrs->spi = ntohl(savp->spi); 196 attrs->family = saidx->dst.sa.sa_family; 197 attrs->reqid = saidx->reqid; 198 199 if (saidx->src.sa.sa_family == AF_INET) { 200 attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr; 201 attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr; 202 } else { 203 memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16); 204 memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16); 205 } 206 207 if (savp->natt) { 208 attrs->encap = true; 209 attrs->sport = savp->natt->sport; 210 attrs->dport = savp->natt->dport; 211 } 212 213 if (savp->flags & SADB_X_SAFLAGS_ESN) { 214 /* We support replay window with ESN only */ 215 attrs->replay_esn.trigger = true; 216 if (sa_entry->esn_state.esn_msb) 217 attrs->replay_esn.esn = sa_entry->esn_state.esn; 218 else 219 /* According to RFC4303, section "3.3.3. Sequence Number Generation", 220 * the first packet sent using a given SA will contain a sequence 221 * number of 1. 222 */ 223 attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1); 224 attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb; 225 attrs->replay_esn.overlap = sa_entry->esn_state.overlap; 226 227 if (savp->replay) { 228 switch (savp->replay->wsize) { 229 case 4: 230 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT; 231 break; 232 case 8: 233 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT; 234 break; 235 case 16: 236 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT; 237 break; 238 case 32: 239 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT; 240 break; 241 default: 242 /* Do nothing */ 243 break; 244 } 245 } 246 } 247 } 248 249 static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, 250 struct secasvar *savp) 251 { 252 struct secasindex *saidx = &savp->sah->saidx; 253 struct seckey *key_encp = savp->key_enc; 254 int keylen; 255 256 if (!(mlx5_ipsec_device_caps(mdev) & 257 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 258 mlx5_core_err(mdev, "FULL offload is not supported\n"); 259 return (EINVAL); 260 } 261 if (savp->state == SADB_SASTATE_DEAD) 262 return (EINVAL); 263 if (savp->alg_enc == SADB_EALG_NONE) { 264 mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n"); 265 return (EINVAL); 266 } 267 if (savp->alg_enc != SADB_X_EALG_AESGCM16) { 268 mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n"); 269 return (EINVAL); 270 } 271 if (savp->tdb_compalgxform) { 272 mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n"); 273 return (EINVAL); 274 } 275 if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) { 276 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n"); 277 return (EINVAL); 278 } 279 if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) || 280 (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) { 281 mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n"); 282 return (EINVAL); 283 } 284 if (saidx->proto != IPPROTO_ESP) { 285 mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n"); 286 return (EINVAL); 287 } 288 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 289 keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 290 if (keylen != 128/8 && keylen != 256 / 8) { 291 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 292 return (EINVAL); 293 } 294 295 if (saidx->mode != IPSEC_MODE_TRANSPORT) { 296 mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n"); 297 return (EINVAL); 298 } 299 300 if (savp->natt) { 301 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) { 302 mlx5_core_err(mdev, "Encapsulation is not supported\n"); 303 return (EINVAL); 304 } 305 } 306 307 if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 && 308 savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) { 309 mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize); 310 return (EINVAL); 311 } 312 313 if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) { 314 if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) { 315 mlx5_core_err(mdev, "ESN is not supported\n"); 316 return (EINVAL); 317 } 318 } else if (savp->replay != NULL && savp->replay->wsize != 0) { 319 mlx5_core_warn(mdev, 320 "non-ESN but replay-protect SA offload is not supported\n"); 321 return (EINVAL); 322 } 323 return 0; 324 } 325 326 static int 327 mlx5e_if_sa_newkey_onedir(struct ifnet *ifp, void *sav, int dir, u_int drv_spi, 328 struct mlx5e_ipsec_sa_entry **privp, struct mlx5e_ipsec_priv_bothdir *pb, 329 struct ifnet *ifpo) 330 { 331 #ifdef IPSEC_OFFLOAD 332 struct rm_priotracker tracker; 333 #endif 334 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 335 struct mlx5e_priv *priv = if_getsoftc(ifp); 336 struct mlx5_core_dev *mdev = priv->mdev; 337 struct mlx5e_ipsec *ipsec = priv->ipsec; 338 u16 vid = VLAN_NONE; 339 int err; 340 341 if (priv->gone != 0 || ipsec == NULL) 342 return (EOPNOTSUPP); 343 344 if (if_gettype(ifpo) == IFT_L2VLAN) 345 VLAN_TAG(ifpo, &vid); 346 347 #ifdef IPSEC_OFFLOAD 348 ipsec_sahtree_rlock(&tracker); 349 #endif 350 err = mlx5e_xfrm_validate_state(mdev, sav); 351 #ifdef IPSEC_OFFLOAD 352 ipsec_sahtree_runlock(&tracker); 353 #endif 354 if (err) 355 return err; 356 357 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 358 if (sa_entry == NULL) 359 return (ENOMEM); 360 361 sa_entry->kspi = drv_spi; 362 sa_entry->savp = sav; 363 sa_entry->ifp = ifp; 364 sa_entry->ifpo = ifpo; 365 sa_entry->ipsec = ipsec; 366 sa_entry->vid = vid; 367 368 #ifdef IPSEC_OFFLOAD 369 ipsec_sahtree_rlock(&tracker); 370 #endif 371 err = mlx5e_xfrm_validate_state(mdev, sav); 372 if (err != 0) { 373 #ifdef IPSEC_OFFLOAD 374 ipsec_sahtree_runlock(&tracker); 375 #endif 376 goto err_xfrm; 377 } 378 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir); 379 #ifdef IPSEC_OFFLOAD 380 ipsec_sahtree_runlock(&tracker); 381 #endif 382 383 err = mlx5e_ipsec_create_dwork(sa_entry, pb); 384 if (err) 385 goto err_xfrm; 386 387 /* create hw context */ 388 err = mlx5_ipsec_create_sa_ctx(sa_entry); 389 if (err) 390 goto err_sa_ctx; 391 392 err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); 393 if (err) 394 goto err_fs; 395 396 *privp = sa_entry; 397 if (sa_entry->dwork) 398 queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED); 399 400 err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL); 401 if (err) 402 goto err_xa; 403 404 return 0; 405 406 err_xa: 407 if (sa_entry->dwork) 408 cancel_delayed_work_sync(&sa_entry->dwork->dwork); 409 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 410 err_fs: 411 mlx5_ipsec_free_sa_ctx(sa_entry); 412 err_sa_ctx: 413 kfree(sa_entry->dwork); 414 sa_entry->dwork = NULL; 415 err_xfrm: 416 kfree(sa_entry); 417 mlx5_en_err(ifp, "Device failed to offload this state"); 418 return err; 419 } 420 421 #define GET_TRUNK_IF(vifp, ifp, ept) \ 422 if (if_gettype(vifp) == IFT_L2VLAN) { \ 423 NET_EPOCH_ENTER(ept); \ 424 ifp = VLAN_TRUNKDEV(vifp); \ 425 NET_EPOCH_EXIT(ept); \ 426 } else { \ 427 ifp = vifp; \ 428 } 429 430 static int 431 mlx5e_if_sa_newkey(struct ifnet *ifpo, void *sav, u_int dev_spi, void **privp) 432 { 433 struct mlx5e_ipsec_priv_bothdir *pb; 434 struct epoch_tracker et; 435 struct ifnet *ifp; 436 int error; 437 438 GET_TRUNK_IF(ifpo, ifp, et); 439 440 pb = malloc(sizeof(struct mlx5e_ipsec_priv_bothdir), M_DEVBUF, 441 M_WAITOK | M_ZERO); 442 error = mlx5e_if_sa_newkey_onedir( 443 ifp, sav, IPSEC_DIR_INBOUND, dev_spi, &pb->priv_in, pb, ifpo); 444 if (error != 0) { 445 free(pb, M_DEVBUF); 446 return (error); 447 } 448 error = mlx5e_if_sa_newkey_onedir( 449 ifp, sav, IPSEC_DIR_OUTBOUND, dev_spi, &pb->priv_out, pb, ifpo); 450 if (error == 0) { 451 *privp = pb; 452 } else { 453 if (pb->priv_in->dwork != NULL) 454 cancel_delayed_work_sync(&pb->priv_in->dwork->dwork); 455 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb->priv_in); 456 free(pb, M_DEVBUF); 457 } 458 return (error); 459 } 460 461 static void 462 mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, void *priv) 463 { 464 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 465 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 466 struct mlx5e_ipsec_sa_entry *old; 467 468 old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id); 469 WARN_ON(old != sa_entry); 470 471 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 472 mlx5_ipsec_free_sa_ctx(sa_entry); 473 kfree(sa_entry->dwork); 474 kfree(sa_entry); 475 } 476 477 static int 478 mlx5e_if_sa_deinstall(struct ifnet *ifpo, u_int dev_spi, void *priv) 479 { 480 struct mlx5e_ipsec_priv_bothdir pb, *pbp; 481 struct epoch_tracker et; 482 struct ifnet *ifp; 483 484 GET_TRUNK_IF(ifpo, ifp, et); 485 486 pbp = priv; 487 pb = *(struct mlx5e_ipsec_priv_bothdir *)priv; 488 pbp->priv_in = pbp->priv_out = NULL; 489 490 if (pb.priv_in->dwork != NULL) 491 cancel_delayed_work_sync(&pb.priv_in->dwork->dwork); 492 if (pb.priv_out->dwork != NULL) 493 cancel_delayed_work_sync(&pb.priv_out->dwork->dwork); 494 495 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_in); 496 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_out); 497 free(pbp, M_DEVBUF); 498 return (0); 499 } 500 501 static void 502 mlx5e_if_sa_cnt_one(struct ifnet *ifp, void *sa, uint32_t drv_spi, 503 void *priv, u64 *bytes, u64 *packets) 504 { 505 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 506 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 507 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 508 509 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 510 } 511 512 static int 513 mlx5e_if_sa_cnt(struct ifnet *ifpo, void *sa, uint32_t drv_spi, void *priv, 514 struct seclifetime *lt) 515 { 516 struct mlx5e_ipsec_priv_bothdir *pb; 517 u64 packets_in, packets_out; 518 u64 bytes_in, bytes_out; 519 struct epoch_tracker et; 520 struct ifnet *ifp; 521 522 GET_TRUNK_IF(ifpo, ifp, et); 523 524 pb = priv; 525 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_in, 526 &bytes_in, &packets_in); 527 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_out, 528 &bytes_out, &packets_out); 529 /* TODO: remove this casting once Kostia changes allocation type to be u64 */ 530 lt->bytes = bytes_in + bytes_out; 531 lt->allocations = (uint32_t)(packets_in + packets_out); 532 return (0); 533 } 534 535 static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, 536 struct secpolicy *sp, struct inpcb *inp) 537 { 538 struct secpolicyindex *spidx = &sp->spidx; 539 540 if (!(mlx5_ipsec_device_caps(mdev) & 541 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 542 mlx5_core_err(mdev, "FULL offload is not supported\n"); 543 return (EINVAL); 544 } 545 546 if (sp->tcount > 1) { 547 mlx5_core_err(mdev, "Can offload exactly one template, " 548 "not %d\n", sp->tcount); 549 return (EINVAL); 550 } 551 552 if (sp->policy == IPSEC_POLICY_BYPASS && 553 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) { 554 mlx5_core_err(mdev, "Device does not support policy priority\n"); 555 return (EINVAL); 556 } 557 558 if (sp->tcount > 0 && inp != NULL) { 559 mlx5_core_err(mdev, "Not valid input data\n"); 560 return (EINVAL); 561 } 562 563 if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) { 564 mlx5_core_err(mdev, "Wrong policy direction\n"); 565 return (EINVAL); 566 } 567 568 if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) { 569 mlx5_core_err(mdev, "Device supports transport mode only"); 570 return (EINVAL); 571 } 572 573 if (sp->policy != IPSEC_POLICY_DISCARD && 574 sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) { 575 mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n"); 576 return (EINVAL); 577 } 578 579 if (sp->policy == IPSEC_POLICY_BYPASS && !inp) { 580 mlx5_core_err(mdev, "Missing port information for IKE bypass\n"); 581 return (EINVAL); 582 } 583 584 if (inp != NULL) { 585 INP_RLOCK(inp); 586 if (inp->inp_socket == NULL || inp->inp_socket->so_proto-> 587 pr_protocol != IPPROTO_UDP) { 588 mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n", 589 inp->inp_socket == NULL ? -1 : 590 inp->inp_socket->so_proto->pr_protocol); 591 INP_RUNLOCK(inp); 592 return (EINVAL); 593 } 594 INP_RUNLOCK(inp); 595 } 596 597 /* TODO fill relevant bits */ 598 return 0; 599 } 600 601 static void 602 mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, 603 struct mlx5_accel_pol_xfrm_attrs *attrs, struct inpcb *inp, u16 vid) 604 { 605 struct secpolicy *sp = pol_entry->sp; 606 struct secpolicyindex *spidx = &sp->spidx; 607 608 memset(attrs, 0, sizeof(*attrs)); 609 610 if (!inp) { 611 if (spidx->src.sa.sa_family == AF_INET) { 612 attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr; 613 attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr; 614 } else if (spidx->src.sa.sa_family == AF_INET6) { 615 memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16); 616 memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16); 617 } else { 618 KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family)); 619 } 620 attrs->family = spidx->src.sa.sa_family; 621 attrs->prio = 0; 622 attrs->action = sp->policy; 623 attrs->reqid = sp->req[0]->saidx.reqid; 624 } else { 625 INP_RLOCK(inp); 626 if ((inp->inp_vflag & INP_IPV4) != 0) { 627 attrs->saddr.a4 = inp->inp_laddr.s_addr; 628 attrs->daddr.a4 = inp->inp_faddr.s_addr; 629 attrs->family = AF_INET; 630 } else if ((inp->inp_vflag & INP_IPV6) != 0) { 631 memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16); 632 memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16); 633 attrs->family = AF_INET6; 634 } else { 635 KASSERT(0, ("unsupported family %d", inp->inp_vflag)); 636 } 637 attrs->upspec.dport = inp->inp_fport; 638 attrs->upspec.sport = inp->inp_lport; 639 attrs->upspec.proto = inp->inp_ip_p; 640 INP_RUNLOCK(inp); 641 642 /* Give highest priority for PCB policies */ 643 attrs->prio = 1; 644 attrs->action = IPSEC_POLICY_IPSEC; 645 } 646 attrs->dir = spidx->dir; 647 attrs->vid = vid; 648 } 649 650 static int 651 mlx5e_if_spd_install(struct ifnet *ifpo, void *sp, void *inp1, void **ifdatap) 652 { 653 struct mlx5e_ipsec_pol_entry *pol_entry; 654 struct mlx5e_priv *priv; 655 struct epoch_tracker et; 656 u16 vid = VLAN_NONE; 657 struct ifnet *ifp; 658 int err; 659 660 GET_TRUNK_IF(ifpo, ifp, et); 661 if (if_gettype(ifpo) == IFT_L2VLAN) 662 VLAN_TAG(ifpo, &vid); 663 priv = if_getsoftc(ifp); 664 if (priv->gone || !priv->ipsec) 665 return (EOPNOTSUPP); 666 667 err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1); 668 if (err) 669 return err; 670 671 pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); 672 if (!pol_entry) 673 return (ENOMEM); 674 675 pol_entry->sp = sp; 676 pol_entry->ipsec = priv->ipsec; 677 678 mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, 679 inp1, vid); 680 err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); 681 if (err) 682 goto err_pol; 683 *ifdatap = pol_entry; 684 685 return 0; 686 687 err_pol: 688 kfree(pol_entry); 689 mlx5_en_err(ifp, "Device failed to offload this policy"); 690 return err; 691 } 692 693 static int 694 mlx5e_if_spd_deinstall(struct ifnet *ifpo, void *sp, void *ifdata) 695 { 696 struct mlx5e_ipsec_pol_entry *pol_entry; 697 698 pol_entry = to_ipsec_pol_entry(ifdata); 699 mlx5e_accel_ipsec_fs_del_pol(pol_entry); 700 kfree(pol_entry); 701 return 0; 702 } 703 704 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 705 { 706 struct mlx5e_ipsec *pipsec = priv->ipsec; 707 if (!pipsec) 708 return; 709 710 mlx5e_accel_ipsec_fs_cleanup(pipsec); 711 destroy_workqueue(pipsec->wq); 712 mlx5e_ipsec_aso_cleanup(pipsec); 713 kfree(pipsec); 714 priv->ipsec = NULL; 715 } 716 717 static int 718 mlx5e_if_ipsec_hwassist(if_t ifneto, void *sav __unused, 719 uint32_t drv_spi __unused, void *priv __unused) 720 { 721 if_t ifnet; 722 723 if (if_gettype(ifneto) == IFT_L2VLAN) { 724 ifnet = VLAN_TRUNKDEV(ifneto); 725 } else { 726 ifnet = ifneto; 727 } 728 729 return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP | 730 CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP)); 731 } 732 733 static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = { 734 .if_sa_newkey = mlx5e_if_sa_newkey, 735 .if_sa_deinstall = mlx5e_if_sa_deinstall, 736 .if_spdadd = mlx5e_if_spd_install, 737 .if_spddel = mlx5e_if_spd_deinstall, 738 .if_sa_cnt = mlx5e_if_sa_cnt, 739 .if_hwassist = mlx5e_if_ipsec_hwassist, 740 }; 741 742 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 743 { 744 struct mlx5_core_dev *mdev = priv->mdev; 745 struct mlx5e_ipsec *pipsec; 746 if_t ifp = priv->ifp; 747 int ret; 748 749 mlx5_core_info(mdev, "ipsec " 750 "offload %d log_max_dek %d gen_obj_types %d " 751 "ipsec_encrypt %d ipsec_decrypt %d " 752 "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d " 753 "ipsec_full_offload %d " 754 "reformat_add_esp_trasport %d reformat_del_esp_trasport %d " 755 "decap %d " 756 "ignore_flow_level_tx %d ignore_flow_level_rx %d " 757 "reformat_natt_tx %d reformat_natt_rx %d " 758 "ipsec_esn %d\n", 759 MLX5_CAP_GEN(mdev, ipsec_offload) != 0, 760 MLX5_CAP_GEN(mdev, log_max_dek) != 0, 761 (MLX5_CAP_GEN_64(mdev, general_obj_types) & 762 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0, 763 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0, 764 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0, 765 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0, 766 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0, 767 MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0, 768 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0, 769 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0, 770 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0, 771 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0, 772 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0, 773 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, 774 reformat_add_esp_transport_over_udp) != 0, 775 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 776 reformat_del_esp_transport_over_udp) != 0, 777 MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0); 778 779 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 780 mlx5_core_dbg(mdev, "Not an IPSec offload device\n"); 781 return 0; 782 } 783 784 xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC); 785 786 pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL); 787 if (pipsec == NULL) 788 return (ENOMEM); 789 790 pipsec->mdev = mdev; 791 pipsec->pdn = priv->pdn; 792 pipsec->mkey = priv->mr.key; 793 794 ret = mlx5e_ipsec_aso_init(pipsec); 795 if (ret) 796 goto err_ipsec_aso; 797 798 pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0); 799 if (pipsec->wq == NULL) { 800 ret = ENOMEM; 801 goto err_ipsec_wq; 802 } 803 804 ret = mlx5e_accel_ipsec_fs_init(pipsec); 805 if (ret) 806 goto err_ipsec_alloc; 807 808 if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs); 809 priv->ipsec = pipsec; 810 mlx5_core_dbg(mdev, "IPSec attached to netdevice\n"); 811 return 0; 812 813 err_ipsec_alloc: 814 destroy_workqueue(pipsec->wq); 815 err_ipsec_wq: 816 mlx5e_ipsec_aso_cleanup(pipsec); 817 err_ipsec_aso: 818 kfree(pipsec); 819 mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret); 820 return ret; 821 } 822