1 /*- 2 * Copyright (c) 2023 NVIDIA corporation & affiliates. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include "opt_ipsec.h" 28 29 #include <sys/types.h> 30 #include <netinet/in.h> 31 #include <sys/socket.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <net/if.h> 35 #include <net/if_var.h> 36 #include <net/pfkeyv2.h> 37 #include <netipsec/key.h> 38 #include <netipsec/key_var.h> 39 #include <netipsec/keydb.h> 40 #include <netipsec/ipsec.h> 41 #include <netipsec/xform.h> 42 #include <netipsec/ipsec_offload.h> 43 #include <dev/mlx5/fs.h> 44 #include <dev/mlx5/mlx5_en/en.h> 45 #include <dev/mlx5/mlx5_accel/ipsec.h> 46 47 #define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000) 48 49 static void mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, 50 void *priv); 51 static int mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv); 52 53 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x) 54 { 55 return (struct mlx5e_ipsec_sa_entry *)x; 56 } 57 58 static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x) 59 { 60 return (struct mlx5e_ipsec_pol_entry *)x; 61 } 62 63 static void 64 mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry *sa_entry, 65 u64 *packets, u64 *bytes) 66 { 67 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 68 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 69 70 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 71 } 72 73 static struct mlx5e_ipsec_sa_entry * 74 mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir *pb, 75 struct mlx5e_ipsec_sa_entry *sa_entry) 76 { 77 return (pb->priv_in == sa_entry ? pb->priv_out : pb->priv_in); 78 } 79 80 static void 81 mlx5e_ipsec_handle_counters(struct work_struct *_work) 82 { 83 struct mlx5e_ipsec_dwork *dwork = 84 container_of(_work, struct mlx5e_ipsec_dwork, dwork.work); 85 struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry; 86 struct mlx5e_ipsec_sa_entry *other_sa_entry; 87 u64 bytes, bytes1, packets1, packets; 88 89 if (sa_entry->attrs.drop) 90 return; 91 other_sa_entry = mlx5e_ipsec_other_sa_entry(dwork->pb, sa_entry); 92 if (other_sa_entry == NULL || other_sa_entry->attrs.drop) 93 return; 94 95 mlx5e_ipsec_handle_counters_onedir(sa_entry, &packets, &bytes); 96 mlx5e_ipsec_handle_counters_onedir(other_sa_entry, &packets1, &bytes1); 97 packets += packets1; 98 bytes += bytes1; 99 100 #ifdef IPSEC_OFFLOAD 101 ipsec_accel_drv_sa_lifetime_update( 102 sa_entry->savp, sa_entry->ifpo, sa_entry->kspi, bytes, packets); 103 #endif 104 105 queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, 106 MLX5_IPSEC_RESCHED); 107 } 108 109 static int 110 mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry, 111 struct mlx5e_ipsec_priv_bothdir *pb) 112 { 113 struct mlx5e_ipsec_dwork *dwork; 114 115 dwork = kzalloc(sizeof(*dwork), GFP_KERNEL); 116 if (!dwork) 117 return (ENOMEM); 118 119 dwork->sa_entry = sa_entry; 120 dwork->pb = pb; 121 INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters); 122 sa_entry->dwork = dwork; 123 return 0; 124 } 125 126 static int mlx5_xform_ah_authsize(const struct auth_hash *esph) 127 { 128 int alen; 129 130 if (esph == NULL) 131 return 0; 132 133 switch (esph->type) { 134 case CRYPTO_SHA2_256_HMAC: 135 case CRYPTO_SHA2_384_HMAC: 136 case CRYPTO_SHA2_512_HMAC: 137 alen = esph->hashsize / 2; /* RFC4868 2.3 */ 138 break; 139 140 case CRYPTO_POLY1305: 141 case CRYPTO_AES_NIST_GMAC: 142 alen = esph->hashsize; 143 break; 144 145 default: 146 alen = AH_HMAC_HASHLEN; 147 break; 148 } 149 150 return alen; 151 } 152 153 void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 154 struct mlx5_accel_esp_xfrm_attrs *attrs, 155 u8 dir) 156 { 157 struct secasvar *savp = sa_entry->savp; 158 const struct auth_hash *esph = savp->tdb_authalgxform; 159 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 160 struct secasindex *saidx = &savp->sah->saidx; 161 struct seckey *key_encap = savp->key_enc; 162 int key_len; 163 164 memset(attrs, 0, sizeof(*attrs)); 165 166 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 167 key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 168 169 memcpy(aes_gcm->aes_key, key_encap->key_data, key_len); 170 aes_gcm->key_len = key_len; 171 172 /* salt and seq_iv */ 173 aes_gcm->seq_iv = 0; 174 memcpy(&aes_gcm->salt, key_encap->key_data + key_len, 175 sizeof(aes_gcm->salt)); 176 177 switch (savp->alg_enc) { 178 case SADB_X_EALG_AESGCM8: 179 attrs->authsize = 8 / 4; /* in dwords */ 180 break; 181 case SADB_X_EALG_AESGCM12: 182 attrs->authsize = 12 / 4; /* in dwords */ 183 break; 184 case SADB_X_EALG_AESGCM16: 185 attrs->authsize = 16 / 4; /* in dwords */ 186 break; 187 default: break; 188 } 189 190 /* iv len */ 191 aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense 192 193 attrs->dir = dir; 194 /* spi - host order */ 195 attrs->spi = ntohl(savp->spi); 196 attrs->family = saidx->dst.sa.sa_family; 197 attrs->reqid = saidx->reqid; 198 199 if (saidx->src.sa.sa_family == AF_INET) { 200 attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr; 201 attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr; 202 } else { 203 memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16); 204 memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16); 205 } 206 207 if (savp->natt) { 208 attrs->encap = true; 209 attrs->sport = savp->natt->sport; 210 attrs->dport = savp->natt->dport; 211 } 212 213 if (savp->flags & SADB_X_SAFLAGS_ESN) { 214 /* We support replay window with ESN only */ 215 attrs->replay_esn.trigger = true; 216 if (sa_entry->esn_state.esn_msb) 217 attrs->replay_esn.esn = sa_entry->esn_state.esn; 218 else 219 /* According to RFC4303, section "3.3.3. Sequence Number Generation", 220 * the first packet sent using a given SA will contain a sequence 221 * number of 1. 222 */ 223 attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1); 224 attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb; 225 attrs->replay_esn.overlap = sa_entry->esn_state.overlap; 226 227 if (savp->replay) { 228 switch (savp->replay->wsize) { 229 case 4: 230 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT; 231 break; 232 case 8: 233 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT; 234 break; 235 case 16: 236 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT; 237 break; 238 case 32: 239 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT; 240 break; 241 default: 242 /* Do nothing */ 243 break; 244 } 245 } 246 } 247 } 248 249 static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, 250 struct secasvar *savp) 251 { 252 struct secasindex *saidx = &savp->sah->saidx; 253 struct seckey *key_encp = savp->key_enc; 254 int keylen; 255 256 if (!(mlx5_ipsec_device_caps(mdev) & 257 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 258 mlx5_core_err(mdev, "FULL offload is not supported\n"); 259 return (EOPNOTSUPP); 260 } 261 if (savp->state == SADB_SASTATE_DEAD) 262 return (EOPNOTSUPP); 263 if (savp->alg_enc == SADB_EALG_NONE) { 264 mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n"); 265 return (EOPNOTSUPP); 266 } 267 if (savp->alg_enc != SADB_X_EALG_AESGCM16) { 268 mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n"); 269 return (EOPNOTSUPP); 270 } 271 if (savp->tdb_compalgxform) { 272 mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n"); 273 return (EOPNOTSUPP); 274 } 275 if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) { 276 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n"); 277 return (EOPNOTSUPP); 278 } 279 if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) || 280 (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) { 281 mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n"); 282 return (EOPNOTSUPP); 283 } 284 if (saidx->proto != IPPROTO_ESP) { 285 mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n"); 286 return (EOPNOTSUPP); 287 } 288 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 289 keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4; 290 if (keylen != 128/8 && keylen != 256 / 8) { 291 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 292 return (EOPNOTSUPP); 293 } 294 295 if (saidx->mode != IPSEC_MODE_TRANSPORT) { 296 mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offload mode\n"); 297 return (EOPNOTSUPP); 298 } 299 300 if (savp->natt) { 301 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) { 302 mlx5_core_err(mdev, "Encapsulation is not supported\n"); 303 return (EOPNOTSUPP); 304 } 305 } 306 307 if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 && 308 savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) { 309 mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize); 310 return (EOPNOTSUPP); 311 } 312 313 if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) { 314 if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) { 315 mlx5_core_err(mdev, "ESN is not supported\n"); 316 return (EOPNOTSUPP); 317 } 318 } else if (savp->replay != NULL && savp->replay->wsize != 0) { 319 mlx5_core_warn(mdev, 320 "non-ESN but replay-protect SA offload is not supported\n"); 321 return (EOPNOTSUPP); 322 } 323 return 0; 324 } 325 326 static int 327 mlx5e_if_sa_newkey_onedir(struct ifnet *ifp, void *sav, int dir, u_int drv_spi, 328 struct mlx5e_ipsec_sa_entry **privp, struct mlx5e_ipsec_priv_bothdir *pb, 329 struct ifnet *ifpo) 330 { 331 #ifdef IPSEC_OFFLOAD 332 struct rm_priotracker tracker; 333 #endif 334 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 335 struct mlx5e_priv *priv = if_getsoftc(ifp); 336 struct mlx5_core_dev *mdev = priv->mdev; 337 struct mlx5e_ipsec *ipsec = priv->ipsec; 338 u16 vid = VLAN_NONE; 339 int err; 340 341 if (priv->gone != 0 || ipsec == NULL) 342 return (EOPNOTSUPP); 343 344 if (if_gettype(ifpo) == IFT_L2VLAN) 345 VLAN_TAG(ifpo, &vid); 346 347 #ifdef IPSEC_OFFLOAD 348 ipsec_sahtree_rlock(&tracker); 349 #endif 350 err = mlx5e_xfrm_validate_state(mdev, sav); 351 #ifdef IPSEC_OFFLOAD 352 ipsec_sahtree_runlock(&tracker); 353 #endif 354 if (err) 355 return err; 356 357 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 358 if (sa_entry == NULL) 359 return (ENOMEM); 360 361 sa_entry->kspi = drv_spi; 362 sa_entry->savp = sav; 363 sa_entry->ifp = ifp; 364 sa_entry->ifpo = ifpo; 365 sa_entry->ipsec = ipsec; 366 sa_entry->vid = vid; 367 368 #ifdef IPSEC_OFFLOAD 369 ipsec_sahtree_rlock(&tracker); 370 #endif 371 err = mlx5e_xfrm_validate_state(mdev, sav); 372 if (err != 0) { 373 #ifdef IPSEC_OFFLOAD 374 ipsec_sahtree_runlock(&tracker); 375 #endif 376 goto err_xfrm; 377 } 378 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir); 379 #ifdef IPSEC_OFFLOAD 380 ipsec_sahtree_runlock(&tracker); 381 #endif 382 383 err = mlx5e_ipsec_create_dwork(sa_entry, pb); 384 if (err) 385 goto err_xfrm; 386 387 /* create hw context */ 388 err = mlx5_ipsec_create_sa_ctx(sa_entry); 389 if (err) 390 goto err_sa_ctx; 391 392 err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); 393 if (err) 394 goto err_fs; 395 396 *privp = sa_entry; 397 if (sa_entry->dwork) 398 queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED); 399 400 err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL); 401 if (err) 402 goto err_xa; 403 404 return 0; 405 406 err_xa: 407 if (sa_entry->dwork) 408 cancel_delayed_work_sync(&sa_entry->dwork->dwork); 409 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 410 err_fs: 411 mlx5_ipsec_free_sa_ctx(sa_entry); 412 err_sa_ctx: 413 kfree(sa_entry->dwork); 414 sa_entry->dwork = NULL; 415 err_xfrm: 416 kfree(sa_entry); 417 mlx5_en_err(ifp, "Device failed to offload this state"); 418 return err; 419 } 420 421 static struct ifnet * 422 mlx5_get_trunk_if(struct ifnet *vifp) 423 { 424 struct epoch_tracker et; 425 struct ifnet *res; 426 427 if (if_gettype(vifp) == IFT_L2VLAN) { 428 NET_EPOCH_ENTER(et); 429 res = VLAN_TRUNKDEV(vifp); 430 NET_EPOCH_EXIT(et); 431 } else { 432 res = vifp; 433 } 434 return (res); 435 } 436 437 static int 438 mlx5e_if_sa_newkey(struct ifnet *ifpo, void *sav, u_int dev_spi, void **privp) 439 { 440 struct mlx5e_ipsec_priv_bothdir *pb; 441 struct ifnet *ifp; 442 int error; 443 444 ifp = mlx5_get_trunk_if(ifpo); 445 446 pb = malloc(sizeof(struct mlx5e_ipsec_priv_bothdir), M_DEVBUF, 447 M_WAITOK | M_ZERO); 448 error = mlx5e_if_sa_newkey_onedir( 449 ifp, sav, IPSEC_DIR_INBOUND, dev_spi, &pb->priv_in, pb, ifpo); 450 if (error != 0) { 451 free(pb, M_DEVBUF); 452 return (error); 453 } 454 error = mlx5e_if_sa_newkey_onedir( 455 ifp, sav, IPSEC_DIR_OUTBOUND, dev_spi, &pb->priv_out, pb, ifpo); 456 if (error == 0) { 457 *privp = pb; 458 } else { 459 if (pb->priv_in->dwork != NULL) 460 cancel_delayed_work_sync(&pb->priv_in->dwork->dwork); 461 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb->priv_in); 462 free(pb, M_DEVBUF); 463 } 464 return (error); 465 } 466 467 static void 468 mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, void *priv) 469 { 470 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 471 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 472 struct mlx5e_ipsec_sa_entry *old; 473 474 old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id); 475 WARN_ON(old != sa_entry); 476 477 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 478 mlx5_ipsec_free_sa_ctx(sa_entry); 479 kfree(sa_entry->dwork); 480 kfree(sa_entry); 481 } 482 483 static int 484 mlx5e_if_sa_deinstall(struct ifnet *ifpo, u_int dev_spi, void *priv) 485 { 486 struct mlx5e_ipsec_priv_bothdir pb, *pbp; 487 struct ifnet *ifp; 488 489 ifp = mlx5_get_trunk_if(ifpo); 490 491 pbp = priv; 492 pb = *(struct mlx5e_ipsec_priv_bothdir *)priv; 493 pbp->priv_in = pbp->priv_out = NULL; 494 495 if (pb.priv_in->dwork != NULL) 496 cancel_delayed_work_sync(&pb.priv_in->dwork->dwork); 497 if (pb.priv_out->dwork != NULL) 498 cancel_delayed_work_sync(&pb.priv_out->dwork->dwork); 499 500 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_in); 501 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_out); 502 free(pbp, M_DEVBUF); 503 return (0); 504 } 505 506 static void 507 mlx5e_if_sa_cnt_one(struct ifnet *ifp, void *sa, uint32_t drv_spi, 508 void *priv, u64 *bytes, u64 *packets) 509 { 510 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv); 511 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 512 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 513 514 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes); 515 } 516 517 static int 518 mlx5e_if_sa_cnt(struct ifnet *ifpo, void *sa, uint32_t drv_spi, void *priv, 519 struct seclifetime *lt) 520 { 521 struct mlx5e_ipsec_priv_bothdir *pb; 522 u64 packets_in, packets_out; 523 u64 bytes_in, bytes_out; 524 struct ifnet *ifp; 525 526 ifp = mlx5_get_trunk_if(ifpo); 527 528 pb = priv; 529 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_in, 530 &bytes_in, &packets_in); 531 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_out, 532 &bytes_out, &packets_out); 533 /* TODO: remove this casting once Kostia changes allocation type to be u64 */ 534 lt->bytes = bytes_in + bytes_out; 535 lt->allocations = (uint32_t)(packets_in + packets_out); 536 return (0); 537 } 538 539 static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, 540 struct secpolicy *sp, struct inpcb *inp) 541 { 542 struct secpolicyindex *spidx = &sp->spidx; 543 544 if (!(mlx5_ipsec_device_caps(mdev) & 545 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 546 mlx5_core_err(mdev, "FULL offload is not supported\n"); 547 return (EINVAL); 548 } 549 550 if (sp->tcount > 1) { 551 mlx5_core_err(mdev, "Can offload exactly one template, " 552 "not %d\n", sp->tcount); 553 return (EINVAL); 554 } 555 556 if (sp->policy == IPSEC_POLICY_BYPASS && 557 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) { 558 mlx5_core_err(mdev, "Device does not support policy priority\n"); 559 return (EINVAL); 560 } 561 562 if (sp->tcount > 0 && inp != NULL) { 563 mlx5_core_err(mdev, "Not valid input data\n"); 564 return (EINVAL); 565 } 566 567 if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) { 568 mlx5_core_err(mdev, "Wrong policy direction\n"); 569 return (EINVAL); 570 } 571 572 if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) { 573 mlx5_core_err(mdev, "Device supports transport mode only"); 574 return (EINVAL); 575 } 576 577 if (sp->policy != IPSEC_POLICY_DISCARD && 578 sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) { 579 mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n"); 580 return (EINVAL); 581 } 582 583 if (sp->policy == IPSEC_POLICY_BYPASS && !inp) { 584 mlx5_core_err(mdev, "Missing port information for IKE bypass\n"); 585 return (EINVAL); 586 } 587 588 if (inp != NULL) { 589 INP_RLOCK(inp); 590 if (inp->inp_socket == NULL || inp->inp_socket->so_proto-> 591 pr_protocol != IPPROTO_UDP) { 592 mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n", 593 inp->inp_socket == NULL ? -1 : 594 inp->inp_socket->so_proto->pr_protocol); 595 INP_RUNLOCK(inp); 596 return (EINVAL); 597 } 598 INP_RUNLOCK(inp); 599 } 600 601 /* TODO fill relevant bits */ 602 return 0; 603 } 604 605 static void 606 mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, 607 struct mlx5_accel_pol_xfrm_attrs *attrs, struct inpcb *inp, u16 vid) 608 { 609 struct secpolicy *sp = pol_entry->sp; 610 struct secpolicyindex *spidx = &sp->spidx; 611 612 memset(attrs, 0, sizeof(*attrs)); 613 614 if (!inp) { 615 if (spidx->src.sa.sa_family == AF_INET) { 616 attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr; 617 attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr; 618 } else if (spidx->src.sa.sa_family == AF_INET6) { 619 memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16); 620 memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16); 621 } else { 622 KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family)); 623 } 624 attrs->family = spidx->src.sa.sa_family; 625 attrs->prio = 0; 626 attrs->action = sp->policy; 627 attrs->reqid = sp->req[0]->saidx.reqid; 628 } else { 629 INP_RLOCK(inp); 630 if ((inp->inp_vflag & INP_IPV4) != 0) { 631 attrs->saddr.a4 = inp->inp_laddr.s_addr; 632 attrs->daddr.a4 = inp->inp_faddr.s_addr; 633 attrs->family = AF_INET; 634 } else if ((inp->inp_vflag & INP_IPV6) != 0) { 635 memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16); 636 memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16); 637 attrs->family = AF_INET6; 638 } else { 639 KASSERT(0, ("unsupported family %d", inp->inp_vflag)); 640 } 641 attrs->upspec.dport = inp->inp_fport; 642 attrs->upspec.sport = inp->inp_lport; 643 attrs->upspec.proto = inp->inp_ip_p; 644 INP_RUNLOCK(inp); 645 646 /* Give highest priority for PCB policies */ 647 attrs->prio = 1; 648 attrs->action = IPSEC_POLICY_IPSEC; 649 } 650 attrs->dir = spidx->dir; 651 attrs->vid = vid; 652 } 653 654 static int 655 mlx5e_if_spd_install(struct ifnet *ifpo, void *sp, void *inp1, void **ifdatap) 656 { 657 struct mlx5e_ipsec_pol_entry *pol_entry; 658 struct mlx5e_priv *priv; 659 u16 vid = VLAN_NONE; 660 struct ifnet *ifp; 661 int err; 662 663 ifp = mlx5_get_trunk_if(ifpo); 664 if (if_gettype(ifpo) == IFT_L2VLAN) 665 VLAN_TAG(ifpo, &vid); 666 priv = if_getsoftc(ifp); 667 if (priv->gone || !priv->ipsec) 668 return (EOPNOTSUPP); 669 670 err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1); 671 if (err) 672 return err; 673 674 pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); 675 if (!pol_entry) 676 return (ENOMEM); 677 678 pol_entry->sp = sp; 679 pol_entry->ipsec = priv->ipsec; 680 681 mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, 682 inp1, vid); 683 err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); 684 if (err) 685 goto err_pol; 686 *ifdatap = pol_entry; 687 688 return 0; 689 690 err_pol: 691 kfree(pol_entry); 692 mlx5_en_err(ifp, "Device failed to offload this policy"); 693 return err; 694 } 695 696 static int 697 mlx5e_if_spd_deinstall(struct ifnet *ifpo, void *sp, void *ifdata) 698 { 699 struct mlx5e_ipsec_pol_entry *pol_entry; 700 701 pol_entry = to_ipsec_pol_entry(ifdata); 702 mlx5e_accel_ipsec_fs_del_pol(pol_entry); 703 kfree(pol_entry); 704 return 0; 705 } 706 707 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 708 { 709 struct mlx5e_ipsec *pipsec = priv->ipsec; 710 if (!pipsec) 711 return; 712 713 mlx5e_accel_ipsec_fs_cleanup(pipsec); 714 destroy_workqueue(pipsec->wq); 715 mlx5e_ipsec_aso_cleanup(pipsec); 716 kfree(pipsec); 717 priv->ipsec = NULL; 718 } 719 720 static int 721 mlx5e_if_ipsec_hwassist(if_t ifneto, void *sav __unused, 722 uint32_t drv_spi __unused, void *priv __unused) 723 { 724 if_t ifnet; 725 726 if (if_gettype(ifneto) == IFT_L2VLAN) { 727 ifnet = VLAN_TRUNKDEV(ifneto); 728 } else { 729 ifnet = ifneto; 730 } 731 732 return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP | 733 CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP)); 734 } 735 736 static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = { 737 .if_sa_newkey = mlx5e_if_sa_newkey, 738 .if_sa_deinstall = mlx5e_if_sa_deinstall, 739 .if_spdadd = mlx5e_if_spd_install, 740 .if_spddel = mlx5e_if_spd_deinstall, 741 .if_sa_cnt = mlx5e_if_sa_cnt, 742 .if_hwassist = mlx5e_if_ipsec_hwassist, 743 }; 744 745 void 746 mlx5e_ipsec_report(struct mlx5e_priv *priv) 747 { 748 struct mlx5_core_dev *mdev; 749 750 if (!bootverbose) 751 return; 752 mdev = priv->mdev; 753 mlx5_core_info(mdev, "ipsec " 754 "offload %d log_max_dek %d gen_obj_types %d " 755 "ipsec_encrypt %d ipsec_decrypt %d " 756 "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d " 757 "ipsec_full_offload %d " 758 "reformat_add_esp_trasport %d reformat_del_esp_trasport %d " 759 "decap %d " 760 "ignore_flow_level_tx %d ignore_flow_level_rx %d " 761 "reformat_natt_tx %d reformat_natt_rx %d " 762 "ipsec_esn %d\n", 763 MLX5_CAP_GEN(mdev, ipsec_offload) != 0, 764 MLX5_CAP_GEN(mdev, log_max_dek) != 0, 765 (MLX5_CAP_GEN_64(mdev, general_obj_types) & 766 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0, 767 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0, 768 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0, 769 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0, 770 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0, 771 MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0, 772 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0, 773 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0, 774 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0, 775 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0, 776 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0, 777 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, 778 reformat_add_esp_transport_over_udp) != 0, 779 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 780 reformat_del_esp_transport_over_udp) != 0, 781 MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0); 782 } 783 784 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 785 { 786 struct mlx5_core_dev *mdev = priv->mdev; 787 struct mlx5e_ipsec *pipsec; 788 if_t ifp = priv->ifp; 789 int ret; 790 791 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { 792 mlx5_core_dbg(mdev, "Not an IPSec offload device\n"); 793 return 0; 794 } 795 796 xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC); 797 798 pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL); 799 if (pipsec == NULL) 800 return (ENOMEM); 801 802 pipsec->mdev = mdev; 803 pipsec->pdn = priv->pdn; 804 pipsec->mkey = priv->mr.key; 805 806 ret = mlx5e_ipsec_aso_init(pipsec); 807 if (ret) 808 goto err_ipsec_aso; 809 810 pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0); 811 if (pipsec->wq == NULL) { 812 ret = ENOMEM; 813 goto err_ipsec_wq; 814 } 815 816 ret = mlx5e_accel_ipsec_fs_init(pipsec); 817 if (ret) 818 goto err_ipsec_alloc; 819 820 if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs); 821 priv->ipsec = pipsec; 822 mlx5_core_dbg(mdev, "IPSec attached to netdevice\n"); 823 return 0; 824 825 err_ipsec_alloc: 826 destroy_workqueue(pipsec->wq); 827 err_ipsec_wq: 828 mlx5e_ipsec_aso_cleanup(pipsec); 829 err_ipsec_aso: 830 kfree(pipsec); 831 mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret); 832 return ret; 833 } 834