1 /*- 2 * Copyright (c) 2019-2021 Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2022 NVIDIA corporation & affiliates. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include "opt_kern_tls.h" 28 #include "opt_rss.h" 29 #include "opt_ratelimit.h" 30 31 #include <dev/mlx5/mlx5_en/en.h> 32 33 #include <dev/mlx5/tls.h> 34 #include <dev/mlx5/crypto.h> 35 36 #include <linux/delay.h> 37 #include <sys/ktls.h> 38 #include <opencrypto/cryptodev.h> 39 40 #ifdef KERN_TLS 41 42 #ifdef RATELIMIT 43 static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify; 44 #endif 45 static if_snd_tag_query_t mlx5e_tls_snd_tag_query; 46 static if_snd_tag_free_t mlx5e_tls_snd_tag_free; 47 48 static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = { 49 .snd_tag_query = mlx5e_tls_snd_tag_query, 50 .snd_tag_free = mlx5e_tls_snd_tag_free, 51 .type = IF_SND_TAG_TYPE_TLS 52 }; 53 54 #ifdef RATELIMIT 55 static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = { 56 .snd_tag_modify = mlx5e_tls_rl_snd_tag_modify, 57 .snd_tag_query = mlx5e_tls_snd_tag_query, 58 .snd_tag_free = mlx5e_tls_snd_tag_free, 59 .type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT 60 }; 61 #endif 62 63 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS"); 64 65 /* software TLS context */ 66 struct mlx5_ifc_sw_tls_cntx_bits { 67 struct mlx5_ifc_tls_static_params_bits param; 68 struct mlx5_ifc_tls_progress_params_bits progress; 69 struct { 70 uint8_t key_data[8][0x20]; 71 uint8_t key_len[0x20]; 72 } key; 73 }; 74 75 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params)); 76 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc)); 77 78 static const char *mlx5e_tls_stats_desc[] = { 79 MLX5E_TLS_STATS(MLX5E_STATS_DESC) 80 }; 81 82 static void mlx5e_tls_work(struct work_struct *); 83 static void mlx5e_tls_prealloc_work(struct work_struct *); 84 85 /* 86 * Expand the tls tag UMA zone in an async context 87 */ 88 89 static void 90 mlx5e_tls_prealloc_work(struct work_struct *work) 91 { 92 struct mlx5e_priv *priv; 93 struct mlx5e_tls *ptls; 94 struct mlx5e_tls_tag **tags; 95 int i, nitems; 96 97 ptls = container_of(work, struct mlx5e_tls, prealloc_work); 98 priv = container_of(ptls, struct mlx5e_priv, tls); 99 nitems = ptls->zone_max; 100 101 tags = malloc(sizeof(tags[0]) * nitems, 102 M_MLX5E_TLS, M_WAITOK | M_ZERO); 103 for (i = 0; i < nitems; i++) { 104 tags[i] = uma_zalloc(priv->tls.zone, M_NOWAIT); 105 /* 106 * If the allocation fails, its likely we are competing 107 * with real consumers of tags and the zone is full, 108 * so exit the loop, and release the tags like we would 109 * if we allocated all "nitems" 110 */ 111 if (tags[i] == NULL) 112 break; 113 } 114 __compiler_membar(); 115 for (i = 0; i < nitems && tags[i] != NULL; i++) 116 uma_zfree(priv->tls.zone, tags[i]); 117 free(tags, M_MLX5E_TLS); 118 } 119 120 static int 121 mlx5e_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags) 122 { 123 struct mlx5e_tls_tag *ptag; 124 struct mlx5e_priv *priv = arg; 125 int err, i; 126 127 /* 128 * mlx5_tls_open_tis() sleeps on a firmware command, so 129 * zone allocations must be done from a sleepable context. 130 * Note that the uma_zalloc() in mlx5e_tls_snd_tag_alloc() 131 * is done with M_NOWAIT so that hitting the zone limit does 132 * not cause the allocation to pause forever. 133 */ 134 135 for (i = 0; i != cnt; i++) { 136 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS, 137 mlx5_dev_domainset(priv->mdev), flags | M_ZERO); 138 if (ptag == NULL) 139 return (i); 140 ptag->tls = &priv->tls; 141 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF); 142 INIT_WORK(&ptag->work, mlx5e_tls_work); 143 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn, 144 priv->pdn, &ptag->tisn); 145 if (err) { 146 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 147 free(ptag, M_MLX5E_TLS); 148 return (i); 149 } 150 151 store[i] = ptag; 152 } 153 return (i); 154 } 155 156 static void 157 mlx5e_tls_tag_release(void *arg, void **store, int cnt) 158 { 159 struct mlx5e_tls_tag *ptag; 160 struct mlx5e_priv *priv; 161 struct mlx5e_tls *ptls; 162 int i; 163 164 for (i = 0; i != cnt; i++) { 165 ptag = store[i]; 166 ptls = ptag->tls; 167 priv = container_of(ptls, struct mlx5e_priv, tls); 168 169 flush_work(&ptag->work); 170 171 if (ptag->tisn != 0) { 172 mlx5_tls_close_tis(priv->mdev, ptag->tisn); 173 } 174 175 mtx_destroy(&ptag->mtx); 176 177 free(ptag, M_MLX5E_TLS); 178 } 179 } 180 181 static void 182 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag) 183 { 184 /* make sure any unhandled taskqueue events are ignored */ 185 ptag->state = MLX5E_TLS_ST_FREED; 186 187 /* reset some variables */ 188 ptag->dek_index = 0; 189 ptag->dek_index_ok = 0; 190 191 /* avoid leaking keys */ 192 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params)); 193 194 /* return tag to UMA */ 195 uma_zfree(ptag->tls->zone, ptag); 196 } 197 198 static int 199 mlx5e_max_tag_proc(SYSCTL_HANDLER_ARGS) 200 { 201 struct mlx5e_priv *priv = (struct mlx5e_priv *)arg1; 202 struct mlx5e_tls *ptls = &priv->tls; 203 int err; 204 unsigned int max_tags; 205 206 max_tags = ptls->zone_max; 207 err = sysctl_handle_int(oidp, &max_tags, arg2, req); 208 if (err != 0 || req->newptr == NULL ) 209 return err; 210 if (max_tags == ptls->zone_max) 211 return 0; 212 if (max_tags > priv->tls.max_resources || max_tags == 0) 213 return (EINVAL); 214 ptls->zone_max = max_tags; 215 uma_zone_set_max(ptls->zone, ptls->zone_max); 216 return 0; 217 } 218 219 int 220 mlx5e_tls_init(struct mlx5e_priv *priv) 221 { 222 struct mlx5e_tls *ptls = &priv->tls; 223 struct sysctl_oid *node; 224 uint32_t max_dek, max_tis, x; 225 int zone_max = 0, prealloc_tags = 0; 226 227 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0 || 228 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0) 229 return (0); 230 231 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq"); 232 if (ptls->wq == NULL) 233 return (ENOMEM); 234 235 sysctl_ctx_init(&ptls->ctx); 236 237 snprintf(ptls->zname, sizeof(ptls->zname), 238 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev)); 239 240 241 TUNABLE_INT_FETCH("hw.mlx5.tls_max_tags", &zone_max); 242 TUNABLE_INT_FETCH("hw.mlx5.tls_prealloc_tags", &prealloc_tags); 243 244 ptls->zone = uma_zcache_create(ptls->zname, 245 sizeof(struct mlx5e_tls_tag), NULL, NULL, NULL, NULL, 246 mlx5e_tls_tag_import, mlx5e_tls_tag_release, priv, 247 UMA_ZONE_UNMANAGED | (prealloc_tags ? UMA_ZONE_NOFREE : 0)); 248 249 /* shared between RX and TX TLS */ 250 max_dek = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1); 251 max_tis = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_tis) - 1); 252 ptls->max_resources = MIN(max_dek, max_tis); 253 254 if (zone_max != 0) { 255 ptls->zone_max = zone_max; 256 if (ptls->zone_max > priv->tls.max_resources) 257 ptls->zone_max = priv->tls.max_resources; 258 } else { 259 ptls->zone_max = priv->tls.max_resources; 260 } 261 262 uma_zone_set_max(ptls->zone, ptls->zone_max); 263 264 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 265 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK); 266 267 ptls->init = 1; 268 269 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, 270 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, 271 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload"); 272 if (node == NULL) 273 return (0); 274 275 SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "tls_max_tag", 276 CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, priv, 0, mlx5e_max_tag_proc, 277 "IU", "Max number of TLS offload session tags"); 278 279 mlx5e_create_counter_stats(&ptls->ctx, 280 SYSCTL_CHILDREN(node), "stats", 281 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM, 282 ptls->stats.arg); 283 284 return (0); 285 } 286 287 void 288 mlx5e_tls_prealloc_tags(struct mlx5e_priv *priv) 289 { 290 struct mlx5e_tls *ptls = &priv->tls; 291 int prealloc_tags = 0; 292 293 if (ptls->prealloc_wq != NULL) 294 return; 295 296 TUNABLE_INT_FETCH("hw.mlx5.tls_prealloc_tags", &prealloc_tags); 297 if (prealloc_tags == 0) 298 return; 299 ptls->prealloc_wq = create_singlethread_workqueue("mlx5-tls-prealloc_wq"); 300 INIT_WORK(&ptls->prealloc_work, mlx5e_tls_prealloc_work); 301 queue_work(ptls->prealloc_wq, &ptls->prealloc_work); 302 } 303 304 void 305 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 306 { 307 struct mlx5e_tls *ptls = &priv->tls; 308 uint32_t x; 309 310 if (ptls->init == 0) 311 return; 312 313 ptls->init = 0; 314 if (ptls->prealloc_wq != NULL) { 315 flush_workqueue(ptls->prealloc_wq); 316 destroy_workqueue(ptls->prealloc_wq); 317 } 318 flush_workqueue(ptls->wq); 319 sysctl_ctx_free(&ptls->ctx); 320 uma_zdestroy(ptls->zone); 321 destroy_workqueue(ptls->wq); 322 323 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 324 counter_u64_free(ptls->stats.arg[x]); 325 } 326 327 328 static int 329 mlx5e_tls_st_init(struct mlx5e_priv *priv, struct mlx5e_tls_tag *ptag) 330 { 331 int err; 332 333 /* try to open TIS, if not present */ 334 if (ptag->tisn == 0) { 335 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn, 336 priv->pdn, &ptag->tisn); 337 if (err) { 338 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 339 return (-err); 340 } 341 } 342 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn); 343 344 /* try to allocate a DEK context ID */ 345 err = mlx5_encryption_key_create(priv->mdev, priv->pdn, 346 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, 347 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data), 348 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len), 349 &ptag->dek_index); 350 if (err) { 351 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 352 return (-err); 353 } 354 355 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index); 356 357 ptag->dek_index_ok = 1; 358 359 MLX5E_TLS_TAG_LOCK(ptag); 360 if (ptag->state == MLX5E_TLS_ST_INIT) 361 ptag->state = MLX5E_TLS_ST_SETUP; 362 MLX5E_TLS_TAG_UNLOCK(ptag); 363 return (0); 364 } 365 366 static void 367 mlx5e_tls_work(struct work_struct *work) 368 { 369 struct mlx5e_tls_tag *ptag; 370 struct mlx5e_priv *priv; 371 372 ptag = container_of(work, struct mlx5e_tls_tag, work); 373 priv = container_of(ptag->tls, struct mlx5e_priv, tls); 374 375 switch (ptag->state) { 376 case MLX5E_TLS_ST_INIT: 377 (void)mlx5e_tls_st_init(priv, ptag); 378 break; 379 380 case MLX5E_TLS_ST_RELEASE: 381 /* try to destroy DEK context by ID */ 382 if (ptag->dek_index_ok) 383 (void)mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index); 384 385 /* free tag */ 386 mlx5e_tls_tag_zfree(ptag); 387 break; 388 389 default: 390 break; 391 } 392 } 393 394 static int 395 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en) 396 { 397 398 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2); 399 if (en->tls_vminor == TLS_MINOR_VER_TWO) 400 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */ 401 else 402 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */ 403 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1); 404 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */ 405 406 /* copy the initial vector in place */ 407 switch (en->iv_len) { 408 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv): 409 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) + 410 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv): 411 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv), 412 en->iv, en->iv_len); 413 break; 414 default: 415 return (EINVAL); 416 } 417 418 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) { 419 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data), 420 en->cipher_key, en->cipher_key_len); 421 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len); 422 } else { 423 return (EINVAL); 424 } 425 return (0); 426 } 427 428 /* Verify zero default */ 429 CTASSERT(MLX5E_TLS_ST_INIT == 0); 430 431 int 432 mlx5e_tls_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params, 433 struct m_snd_tag **ppmt) 434 { 435 union if_snd_tag_alloc_params rl_params; 436 const struct if_snd_tag_sw *snd_tag_sw; 437 struct mlx5e_priv *priv; 438 struct mlx5e_tls_tag *ptag; 439 const struct tls_session_params *en; 440 int error; 441 442 priv = if_getsoftc(ifp); 443 444 if (priv->gone != 0 || priv->tls.init == 0) 445 return (EOPNOTSUPP); 446 447 ptag = uma_zalloc(priv->tls.zone, M_NOWAIT); 448 if (ptag == NULL) 449 return (ENOMEM); 450 451 /* sanity check default values */ 452 MPASS(ptag->dek_index == 0); 453 MPASS(ptag->dek_index_ok == 0); 454 455 /* check if there is no TIS context */ 456 KASSERT(ptag->tisn != 0, ("ptag %p w/0 tisn", ptag)); 457 458 en = ¶ms->tls.tls->params; 459 460 /* only TLS v1.2 and v1.3 is currently supported */ 461 if (en->tls_vmajor != TLS_MAJOR_VER_ONE || 462 (en->tls_vminor != TLS_MINOR_VER_TWO 463 #ifdef TLS_MINOR_VER_THREE 464 && en->tls_vminor != TLS_MINOR_VER_THREE 465 #endif 466 )) { 467 error = EPROTONOSUPPORT; 468 goto failure; 469 } 470 471 switch (en->cipher_algorithm) { 472 case CRYPTO_AES_NIST_GCM_16: 473 switch (en->cipher_key_len) { 474 case 128 / 8: 475 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 476 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) { 477 error = EPROTONOSUPPORT; 478 goto failure; 479 } 480 } else { 481 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) { 482 error = EPROTONOSUPPORT; 483 goto failure; 484 } 485 } 486 error = mlx5e_tls_set_params(ptag->crypto_params, en); 487 if (error) 488 goto failure; 489 break; 490 491 case 256 / 8: 492 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 493 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) { 494 error = EPROTONOSUPPORT; 495 goto failure; 496 } 497 } else { 498 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) { 499 error = EPROTONOSUPPORT; 500 goto failure; 501 } 502 } 503 error = mlx5e_tls_set_params(ptag->crypto_params, en); 504 if (error) 505 goto failure; 506 break; 507 508 default: 509 error = EINVAL; 510 goto failure; 511 } 512 break; 513 default: 514 error = EPROTONOSUPPORT; 515 goto failure; 516 } 517 518 memset(&rl_params, 0, sizeof(rl_params)); 519 rl_params.hdr = params->hdr; 520 switch (params->hdr.type) { 521 #ifdef RATELIMIT 522 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT: 523 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT; 524 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 525 snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw; 526 break; 527 #endif 528 case IF_SND_TAG_TYPE_TLS: 529 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED; 530 snd_tag_sw = &mlx5e_tls_snd_tag_sw; 531 break; 532 default: 533 error = EOPNOTSUPP; 534 goto failure; 535 } 536 537 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag); 538 if (error) 539 goto failure; 540 541 /* store pointer to mbuf tag */ 542 MPASS(ptag->tag.refcount == 0); 543 m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw); 544 *ppmt = &ptag->tag; 545 546 /* reset state */ 547 ptag->state = MLX5E_TLS_ST_INIT; 548 549 error = mlx5e_tls_st_init(priv, ptag); 550 if (error != 0) 551 goto failure; 552 553 return (0); 554 555 failure: 556 mlx5e_tls_tag_zfree(ptag); 557 return (error); 558 } 559 560 #ifdef RATELIMIT 561 static int 562 mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params) 563 { 564 union if_snd_tag_modify_params rl_params; 565 struct mlx5e_tls_tag *ptag = 566 container_of(pmt, struct mlx5e_tls_tag, tag); 567 int error; 568 569 memset(&rl_params, 0, sizeof(rl_params)); 570 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 571 error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params); 572 return (error); 573 } 574 #endif 575 576 static int 577 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params) 578 { 579 struct mlx5e_tls_tag *ptag = 580 container_of(pmt, struct mlx5e_tls_tag, tag); 581 582 return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params)); 583 } 584 585 static void 586 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt) 587 { 588 struct mlx5e_tls_tag *ptag = 589 container_of(pmt, struct mlx5e_tls_tag, tag); 590 struct mlx5e_priv *priv; 591 592 m_snd_tag_rele(ptag->rl_tag); 593 594 MLX5E_TLS_TAG_LOCK(ptag); 595 ptag->state = MLX5E_TLS_ST_RELEASE; 596 MLX5E_TLS_TAG_UNLOCK(ptag); 597 598 priv = if_getsoftc(ptag->tag.ifp); 599 queue_work(priv->tls.wq, &ptag->work); 600 } 601 602 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0); 603 604 static void 605 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 606 { 607 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) + 608 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS); 609 struct mlx5e_tx_umr_wqe *wqe; 610 u16 pi; 611 612 pi = sq->pc & sq->wq.sz_m1; 613 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 614 615 memset(wqe, 0, sizeof(*wqe)); 616 617 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 618 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24)); 619 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 620 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8); 621 622 if (mlx5e_do_send_cqe(sq)) 623 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 624 else 625 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 626 627 /* fill out UMR control segment */ 628 wqe->umr.flags = 0x80; /* inline data */ 629 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16); 630 631 /* copy in the static crypto parameters */ 632 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param), 633 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param)); 634 635 /* copy data for doorbell */ 636 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 637 638 sq->mbuf[pi].mbuf = NULL; 639 sq->mbuf[pi].num_bytes = 0; 640 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 641 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 642 643 sq->pc += sq->mbuf[pi].num_wqebbs; 644 } 645 646 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) == 647 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv)); 648 649 static void 650 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 651 { 652 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe), 653 MLX5_SEND_WQE_DS); 654 struct mlx5e_tx_psv_wqe *wqe; 655 u16 pi; 656 657 pi = sq->pc & sq->wq.sz_m1; 658 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 659 660 memset(wqe, 0, sizeof(*wqe)); 661 662 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 663 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24)); 664 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 665 666 if (mlx5e_do_send_cqe(sq)) 667 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 668 669 /* copy in the PSV control segment */ 670 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress), 671 sizeof(wqe->psv)); 672 673 /* copy data for doorbell */ 674 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 675 676 sq->mbuf[pi].mbuf = NULL; 677 sq->mbuf[pi].num_bytes = 0; 678 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 679 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 680 681 sq->pc += sq->mbuf[pi].num_wqebbs; 682 } 683 684 static void 685 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 686 { 687 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS; 688 struct mlx5e_tx_wqe *wqe; 689 u16 pi; 690 691 pi = sq->pc & sq->wq.sz_m1; 692 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 693 694 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 695 696 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 697 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 698 if (mlx5e_do_send_cqe(sq)) 699 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 700 else 701 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 702 703 /* Copy data for doorbell */ 704 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 705 706 sq->mbuf[pi].mbuf = NULL; 707 sq->mbuf[pi].num_bytes = 0; 708 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 709 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 710 711 sq->pc += sq->mbuf[pi].num_wqebbs; 712 } 713 714 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1) 715 716 static struct mbuf * 717 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start) 718 { 719 struct mbuf *mr, *top; 720 uint32_t offset; 721 uint32_t delta; 722 723 /* check format of incoming mbuf */ 724 if (mb->m_next == NULL || 725 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) { 726 top = NULL; 727 goto done; 728 } 729 730 /* get unmapped data offset */ 731 offset = mtod(mb->m_next, uintptr_t); 732 733 /* check if we don't need to re-transmit anything */ 734 if (offset == 0) { 735 top = SBTLS_MBUF_NO_DATA; 736 *pis_start = true; 737 goto done; 738 } 739 740 /* try to get a new packet header */ 741 top = m_gethdr(wait, MT_DATA); 742 if (top == NULL) 743 goto done; 744 745 mr = m_get(wait, MT_DATA); 746 if (mr == NULL) { 747 m_free(top); 748 top = NULL; 749 goto done; 750 } 751 752 top->m_next = mr; 753 754 mb_dupcl(mr, mb->m_next); 755 756 /* the beginning of the TLS record */ 757 mr->m_data = NULL; 758 759 /* setup packet header length */ 760 top->m_pkthdr.len = mr->m_len = offset; 761 top->m_len = 0; 762 763 /* check for partial re-transmit */ 764 delta = *ptcp_seq - tcp_old; 765 766 if (delta < offset) { 767 m_adj(top, offset - delta); 768 offset = delta; 769 770 /* continue where we left off */ 771 *pis_start = false; 772 } else { 773 *pis_start = true; 774 } 775 776 /* 777 * Rewind the TCP sequence number by the amount of data 778 * retransmitted: 779 */ 780 *ptcp_seq -= offset; 781 done: 782 return (top); 783 } 784 785 static int 786 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq) 787 { 788 789 for (; mb != NULL; mb = mb->m_next) { 790 if (!(mb->m_flags & M_EXTPG)) 791 continue; 792 *pseq = mb->m_epg_seqno; 793 return (1); 794 } 795 return (0); 796 } 797 798 int 799 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb) 800 { 801 struct mlx5e_tls_tag *ptls_tag; 802 struct m_snd_tag *ptag; 803 const struct tcphdr *th; 804 struct mbuf *mb = *ppmb; 805 u64 rcd_sn; 806 u32 header_size; 807 u32 mb_seq; 808 809 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0) 810 return (MLX5E_TLS_CONTINUE); 811 812 ptag = mb->m_pkthdr.snd_tag; 813 814 if ( 815 #ifdef RATELIMIT 816 ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT && 817 #endif 818 ptag->sw->type != IF_SND_TAG_TYPE_TLS) 819 return (MLX5E_TLS_CONTINUE); 820 821 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag); 822 823 header_size = mlx5e_get_full_header_size(mb, &th); 824 if (unlikely(header_size == 0 || th == NULL)) 825 return (MLX5E_TLS_FAILURE); 826 827 /* 828 * Send non-TLS TCP packets AS-IS: 829 */ 830 if (header_size == mb->m_pkthdr.len || 831 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) { 832 parg->tisn = 0; 833 parg->ihs = header_size; 834 return (MLX5E_TLS_CONTINUE); 835 } 836 837 mb_seq = ntohl(th->th_seq); 838 839 MLX5E_TLS_TAG_LOCK(ptls_tag); 840 switch (ptls_tag->state) { 841 case MLX5E_TLS_ST_INIT: 842 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 843 return (MLX5E_TLS_FAILURE); 844 case MLX5E_TLS_ST_SETUP: 845 ptls_tag->state = MLX5E_TLS_ST_TXRDY; 846 ptls_tag->expected_seq = ~mb_seq; /* force setup */ 847 default: 848 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 849 break; 850 } 851 852 if (unlikely(ptls_tag->expected_seq != mb_seq)) { 853 bool is_start; 854 struct mbuf *r_mb; 855 uint32_t tcp_seq = mb_seq; 856 857 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start); 858 if (r_mb == NULL) { 859 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1); 860 return (MLX5E_TLS_FAILURE); 861 } 862 863 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1); 864 865 /* check if this is the first fragment of a TLS record */ 866 if (is_start) { 867 /* setup TLS static parameters */ 868 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params, 869 param.initial_record_number, rcd_sn); 870 871 /* 872 * NOTE: The sendqueue should have enough room to 873 * carry both the static and the progress parameters 874 * when we get here! 875 */ 876 mlx5e_tls_send_static_parameters(sq, ptls_tag); 877 mlx5e_tls_send_progress_parameters(sq, ptls_tag); 878 879 if (r_mb == SBTLS_MBUF_NO_DATA) { 880 mlx5e_tls_send_nop(sq, ptls_tag); 881 ptls_tag->expected_seq = mb_seq; 882 return (MLX5E_TLS_LOOP); 883 } 884 } 885 886 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len); 887 888 /* setup transmit arguments */ 889 parg->tisn = ptls_tag->tisn; 890 parg->mst = &ptls_tag->tag; 891 892 /* try to send DUMP data */ 893 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) { 894 m_freem(r_mb); 895 ptls_tag->expected_seq = tcp_seq; 896 return (MLX5E_TLS_FAILURE); 897 } else { 898 ptls_tag->expected_seq = mb_seq; 899 return (MLX5E_TLS_LOOP); 900 } 901 } else { 902 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1); 903 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len); 904 } 905 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size; 906 907 parg->tisn = ptls_tag->tisn; 908 parg->ihs = header_size; 909 parg->mst = &ptls_tag->tag; 910 return (MLX5E_TLS_CONTINUE); 911 } 912 913 #else 914 915 int 916 mlx5e_tls_init(struct mlx5e_priv *priv) 917 { 918 919 return (0); 920 } 921 922 void 923 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 924 { 925 /* NOP */ 926 } 927 928 #endif /* KERN_TLS */ 929