1 /*- 2 * Copyright (c) 2019-2021 Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2022 NVIDIA corporation & affiliates. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include "opt_kern_tls.h" 28 #include "opt_rss.h" 29 #include "opt_ratelimit.h" 30 31 #include <dev/mlx5/mlx5_en/en.h> 32 33 #include <dev/mlx5/tls.h> 34 #include <dev/mlx5/crypto.h> 35 36 #include <linux/delay.h> 37 #include <sys/ktls.h> 38 #include <opencrypto/cryptodev.h> 39 40 #ifdef KERN_TLS 41 42 #ifdef RATELIMIT 43 static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify; 44 #endif 45 static if_snd_tag_query_t mlx5e_tls_snd_tag_query; 46 static if_snd_tag_free_t mlx5e_tls_snd_tag_free; 47 48 static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = { 49 .snd_tag_query = mlx5e_tls_snd_tag_query, 50 .snd_tag_free = mlx5e_tls_snd_tag_free, 51 .type = IF_SND_TAG_TYPE_TLS 52 }; 53 54 #ifdef RATELIMIT 55 static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = { 56 .snd_tag_modify = mlx5e_tls_rl_snd_tag_modify, 57 .snd_tag_query = mlx5e_tls_snd_tag_query, 58 .snd_tag_free = mlx5e_tls_snd_tag_free, 59 .type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT 60 }; 61 #endif 62 63 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS"); 64 65 /* software TLS context */ 66 struct mlx5_ifc_sw_tls_cntx_bits { 67 struct mlx5_ifc_tls_static_params_bits param; 68 struct mlx5_ifc_tls_progress_params_bits progress; 69 struct { 70 uint8_t key_data[8][0x20]; 71 uint8_t key_len[0x20]; 72 } key; 73 }; 74 75 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params)); 76 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc)); 77 78 static const char *mlx5e_tls_stats_desc[] = { 79 MLX5E_TLS_STATS(MLX5E_STATS_DESC) 80 }; 81 82 static void mlx5e_tls_work(struct work_struct *); 83 84 /* 85 * Expand the tls tag UMA zone in a sleepable context 86 */ 87 88 static void 89 mlx5e_prealloc_tags(struct mlx5e_priv *priv, int nitems) 90 { 91 struct mlx5e_tls_tag **tags; 92 int i; 93 94 tags = malloc(sizeof(tags[0]) * nitems, 95 M_MLX5E_TLS, M_WAITOK); 96 for (i = 0; i < nitems; i++) 97 tags[i] = uma_zalloc(priv->tls.zone, M_WAITOK); 98 __compiler_membar(); 99 for (i = 0; i < nitems; i++) 100 uma_zfree(priv->tls.zone, tags[i]); 101 free(tags, M_MLX5E_TLS); 102 } 103 104 static int 105 mlx5e_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags) 106 { 107 struct mlx5e_tls_tag *ptag; 108 struct mlx5e_priv *priv = arg; 109 int err, i; 110 111 /* 112 * mlx5_tls_open_tis() sleeps on a firmware command, so 113 * zone allocations must be done from a sleepable context. 114 * Note that the uma_zalloc() in mlx5e_tls_snd_tag_alloc() 115 * is done with M_NOWAIT so that hitting the zone limit does 116 * not cause the allocation to pause forever. 117 */ 118 119 for (i = 0; i != cnt; i++) { 120 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS, 121 mlx5_dev_domainset(arg), flags | M_ZERO); 122 if (ptag == NULL) 123 return (i); 124 ptag->tls = &priv->tls; 125 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF); 126 INIT_WORK(&ptag->work, mlx5e_tls_work); 127 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn, 128 priv->pdn, &ptag->tisn); 129 if (err) { 130 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 131 free(ptag, M_MLX5E_TLS); 132 return (i); 133 } 134 135 store[i] = ptag; 136 } 137 return (i); 138 } 139 140 static void 141 mlx5e_tls_tag_release(void *arg, void **store, int cnt) 142 { 143 struct mlx5e_tls_tag *ptag; 144 struct mlx5e_priv *priv; 145 struct mlx5e_tls *ptls; 146 int i; 147 148 for (i = 0; i != cnt; i++) { 149 ptag = store[i]; 150 ptls = ptag->tls; 151 priv = container_of(ptls, struct mlx5e_priv, tls); 152 153 flush_work(&ptag->work); 154 155 if (ptag->tisn != 0) { 156 mlx5_tls_close_tis(priv->mdev, ptag->tisn); 157 } 158 159 mtx_destroy(&ptag->mtx); 160 161 free(ptag, M_MLX5E_TLS); 162 } 163 } 164 165 static void 166 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag) 167 { 168 /* make sure any unhandled taskqueue events are ignored */ 169 ptag->state = MLX5E_TLS_ST_FREED; 170 171 /* reset some variables */ 172 ptag->dek_index = 0; 173 ptag->dek_index_ok = 0; 174 175 /* avoid leaking keys */ 176 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params)); 177 178 /* return tag to UMA */ 179 uma_zfree(ptag->tls->zone, ptag); 180 } 181 182 static int 183 mlx5e_max_tag_proc(SYSCTL_HANDLER_ARGS) 184 { 185 struct mlx5e_priv *priv = (struct mlx5e_priv *)arg1; 186 struct mlx5e_tls *ptls = &priv->tls; 187 int err; 188 unsigned int max_tags; 189 190 max_tags = ptls->zone_max; 191 err = sysctl_handle_int(oidp, &max_tags, arg2, req); 192 if (err != 0 || req->newptr == NULL ) 193 return err; 194 if (max_tags == ptls->zone_max) 195 return 0; 196 if (max_tags > priv->tls.max_resources || max_tags == 0) 197 return (EINVAL); 198 ptls->zone_max = max_tags; 199 uma_zone_set_max(ptls->zone, ptls->zone_max); 200 return 0; 201 } 202 203 int 204 mlx5e_tls_init(struct mlx5e_priv *priv) 205 { 206 struct mlx5e_tls *ptls = &priv->tls; 207 struct sysctl_oid *node; 208 uint32_t max_dek, max_tis, x; 209 int zone_max = 0, prealloc_tags = 0; 210 211 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0 || 212 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0) 213 return (0); 214 215 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq"); 216 if (ptls->wq == NULL) 217 return (ENOMEM); 218 219 sysctl_ctx_init(&ptls->ctx); 220 221 snprintf(ptls->zname, sizeof(ptls->zname), 222 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev)); 223 224 225 TUNABLE_INT_FETCH("hw.mlx5.tls_max_tags", &zone_max); 226 TUNABLE_INT_FETCH("hw.mlx5.tls_prealloc_tags", &prealloc_tags); 227 228 ptls->zone = uma_zcache_create(ptls->zname, 229 sizeof(struct mlx5e_tls_tag), NULL, NULL, NULL, NULL, 230 mlx5e_tls_tag_import, mlx5e_tls_tag_release, priv, 231 UMA_ZONE_UNMANAGED | (prealloc_tags ? UMA_ZONE_NOFREE : 0)); 232 233 /* shared between RX and TX TLS */ 234 max_dek = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1); 235 max_tis = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_tis) - 1); 236 ptls->max_resources = MIN(max_dek, max_tis); 237 238 if (zone_max != 0) { 239 ptls->zone_max = zone_max; 240 if (ptls->zone_max > priv->tls.max_resources) 241 ptls->zone_max = priv->tls.max_resources; 242 } else { 243 ptls->zone_max = priv->tls.max_resources; 244 } 245 246 uma_zone_set_max(ptls->zone, ptls->zone_max); 247 if (prealloc_tags != 0) 248 mlx5e_prealloc_tags(priv, ptls->zone_max); 249 250 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 251 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK); 252 253 ptls->init = 1; 254 255 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, 256 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, 257 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload"); 258 if (node == NULL) 259 return (0); 260 261 SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "tls_max_tag", 262 CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, priv, 0, mlx5e_max_tag_proc, 263 "IU", "Max number of TLS offload session tags"); 264 265 mlx5e_create_counter_stats(&ptls->ctx, 266 SYSCTL_CHILDREN(node), "stats", 267 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM, 268 ptls->stats.arg); 269 270 return (0); 271 } 272 273 void 274 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 275 { 276 struct mlx5e_tls *ptls = &priv->tls; 277 uint32_t x; 278 279 if (ptls->init == 0) 280 return; 281 282 ptls->init = 0; 283 flush_workqueue(ptls->wq); 284 sysctl_ctx_free(&ptls->ctx); 285 uma_zdestroy(ptls->zone); 286 destroy_workqueue(ptls->wq); 287 288 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 289 counter_u64_free(ptls->stats.arg[x]); 290 } 291 292 293 static int 294 mlx5e_tls_st_init(struct mlx5e_priv *priv, struct mlx5e_tls_tag *ptag) 295 { 296 int err; 297 298 /* try to open TIS, if not present */ 299 if (ptag->tisn == 0) { 300 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn, 301 priv->pdn, &ptag->tisn); 302 if (err) { 303 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 304 return (-err); 305 } 306 } 307 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn); 308 309 /* try to allocate a DEK context ID */ 310 err = mlx5_encryption_key_create(priv->mdev, priv->pdn, 311 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, 312 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data), 313 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len), 314 &ptag->dek_index); 315 if (err) { 316 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 317 return (-err); 318 } 319 320 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index); 321 322 ptag->dek_index_ok = 1; 323 324 MLX5E_TLS_TAG_LOCK(ptag); 325 if (ptag->state == MLX5E_TLS_ST_INIT) 326 ptag->state = MLX5E_TLS_ST_SETUP; 327 MLX5E_TLS_TAG_UNLOCK(ptag); 328 return (0); 329 } 330 331 static void 332 mlx5e_tls_work(struct work_struct *work) 333 { 334 struct mlx5e_tls_tag *ptag; 335 struct mlx5e_priv *priv; 336 337 ptag = container_of(work, struct mlx5e_tls_tag, work); 338 priv = container_of(ptag->tls, struct mlx5e_priv, tls); 339 340 switch (ptag->state) { 341 case MLX5E_TLS_ST_INIT: 342 (void)mlx5e_tls_st_init(priv, ptag); 343 break; 344 345 case MLX5E_TLS_ST_RELEASE: 346 /* try to destroy DEK context by ID */ 347 if (ptag->dek_index_ok) 348 (void)mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index); 349 350 /* free tag */ 351 mlx5e_tls_tag_zfree(ptag); 352 break; 353 354 default: 355 break; 356 } 357 } 358 359 static int 360 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en) 361 { 362 363 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2); 364 if (en->tls_vminor == TLS_MINOR_VER_TWO) 365 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */ 366 else 367 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */ 368 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1); 369 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */ 370 371 /* copy the initial vector in place */ 372 switch (en->iv_len) { 373 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv): 374 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) + 375 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv): 376 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv), 377 en->iv, en->iv_len); 378 break; 379 default: 380 return (EINVAL); 381 } 382 383 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) { 384 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data), 385 en->cipher_key, en->cipher_key_len); 386 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len); 387 } else { 388 return (EINVAL); 389 } 390 return (0); 391 } 392 393 /* Verify zero default */ 394 CTASSERT(MLX5E_TLS_ST_INIT == 0); 395 396 int 397 mlx5e_tls_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params, 398 struct m_snd_tag **ppmt) 399 { 400 union if_snd_tag_alloc_params rl_params; 401 const struct if_snd_tag_sw *snd_tag_sw; 402 struct mlx5e_priv *priv; 403 struct mlx5e_tls_tag *ptag; 404 const struct tls_session_params *en; 405 int error; 406 407 priv = if_getsoftc(ifp); 408 409 if (priv->gone != 0 || priv->tls.init == 0) 410 return (EOPNOTSUPP); 411 412 ptag = uma_zalloc(priv->tls.zone, M_WAITOK); 413 if (ptag == NULL) 414 return (ENOMEM); 415 416 /* sanity check default values */ 417 MPASS(ptag->dek_index == 0); 418 MPASS(ptag->dek_index_ok == 0); 419 420 /* check if there is no TIS context */ 421 KASSERT(ptag->tisn != 0, ("ptag %p w/0 tisn", ptag)); 422 423 en = ¶ms->tls.tls->params; 424 425 /* only TLS v1.2 and v1.3 is currently supported */ 426 if (en->tls_vmajor != TLS_MAJOR_VER_ONE || 427 (en->tls_vminor != TLS_MINOR_VER_TWO 428 #ifdef TLS_MINOR_VER_THREE 429 && en->tls_vminor != TLS_MINOR_VER_THREE 430 #endif 431 )) { 432 error = EPROTONOSUPPORT; 433 goto failure; 434 } 435 436 switch (en->cipher_algorithm) { 437 case CRYPTO_AES_NIST_GCM_16: 438 switch (en->cipher_key_len) { 439 case 128 / 8: 440 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 441 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) { 442 error = EPROTONOSUPPORT; 443 goto failure; 444 } 445 } else { 446 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) { 447 error = EPROTONOSUPPORT; 448 goto failure; 449 } 450 } 451 error = mlx5e_tls_set_params(ptag->crypto_params, en); 452 if (error) 453 goto failure; 454 break; 455 456 case 256 / 8: 457 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 458 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) { 459 error = EPROTONOSUPPORT; 460 goto failure; 461 } 462 } else { 463 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) { 464 error = EPROTONOSUPPORT; 465 goto failure; 466 } 467 } 468 error = mlx5e_tls_set_params(ptag->crypto_params, en); 469 if (error) 470 goto failure; 471 break; 472 473 default: 474 error = EINVAL; 475 goto failure; 476 } 477 break; 478 default: 479 error = EPROTONOSUPPORT; 480 goto failure; 481 } 482 483 memset(&rl_params, 0, sizeof(rl_params)); 484 rl_params.hdr = params->hdr; 485 switch (params->hdr.type) { 486 #ifdef RATELIMIT 487 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT: 488 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT; 489 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 490 snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw; 491 break; 492 #endif 493 case IF_SND_TAG_TYPE_TLS: 494 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED; 495 snd_tag_sw = &mlx5e_tls_snd_tag_sw; 496 break; 497 default: 498 error = EOPNOTSUPP; 499 goto failure; 500 } 501 502 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag); 503 if (error) 504 goto failure; 505 506 /* store pointer to mbuf tag */ 507 MPASS(ptag->tag.refcount == 0); 508 m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw); 509 *ppmt = &ptag->tag; 510 511 /* reset state */ 512 ptag->state = MLX5E_TLS_ST_INIT; 513 514 error = mlx5e_tls_st_init(priv, ptag); 515 if (error != 0) 516 goto failure; 517 518 return (0); 519 520 failure: 521 mlx5e_tls_tag_zfree(ptag); 522 return (error); 523 } 524 525 #ifdef RATELIMIT 526 static int 527 mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params) 528 { 529 union if_snd_tag_modify_params rl_params; 530 struct mlx5e_tls_tag *ptag = 531 container_of(pmt, struct mlx5e_tls_tag, tag); 532 int error; 533 534 memset(&rl_params, 0, sizeof(rl_params)); 535 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 536 error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params); 537 return (error); 538 } 539 #endif 540 541 static int 542 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params) 543 { 544 struct mlx5e_tls_tag *ptag = 545 container_of(pmt, struct mlx5e_tls_tag, tag); 546 547 return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params)); 548 } 549 550 static void 551 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt) 552 { 553 struct mlx5e_tls_tag *ptag = 554 container_of(pmt, struct mlx5e_tls_tag, tag); 555 struct mlx5e_priv *priv; 556 557 m_snd_tag_rele(ptag->rl_tag); 558 559 MLX5E_TLS_TAG_LOCK(ptag); 560 ptag->state = MLX5E_TLS_ST_RELEASE; 561 MLX5E_TLS_TAG_UNLOCK(ptag); 562 563 priv = if_getsoftc(ptag->tag.ifp); 564 queue_work(priv->tls.wq, &ptag->work); 565 } 566 567 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0); 568 569 static void 570 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 571 { 572 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) + 573 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS); 574 struct mlx5e_tx_umr_wqe *wqe; 575 u16 pi; 576 577 pi = sq->pc & sq->wq.sz_m1; 578 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 579 580 memset(wqe, 0, sizeof(*wqe)); 581 582 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 583 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24)); 584 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 585 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8); 586 587 if (mlx5e_do_send_cqe(sq)) 588 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 589 else 590 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 591 592 /* fill out UMR control segment */ 593 wqe->umr.flags = 0x80; /* inline data */ 594 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16); 595 596 /* copy in the static crypto parameters */ 597 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param), 598 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param)); 599 600 /* copy data for doorbell */ 601 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 602 603 sq->mbuf[pi].mbuf = NULL; 604 sq->mbuf[pi].num_bytes = 0; 605 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 606 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 607 608 sq->pc += sq->mbuf[pi].num_wqebbs; 609 } 610 611 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) == 612 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv)); 613 614 static void 615 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 616 { 617 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe), 618 MLX5_SEND_WQE_DS); 619 struct mlx5e_tx_psv_wqe *wqe; 620 u16 pi; 621 622 pi = sq->pc & sq->wq.sz_m1; 623 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 624 625 memset(wqe, 0, sizeof(*wqe)); 626 627 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 628 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24)); 629 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 630 631 if (mlx5e_do_send_cqe(sq)) 632 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 633 634 /* copy in the PSV control segment */ 635 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress), 636 sizeof(wqe->psv)); 637 638 /* copy data for doorbell */ 639 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 640 641 sq->mbuf[pi].mbuf = NULL; 642 sq->mbuf[pi].num_bytes = 0; 643 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 644 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 645 646 sq->pc += sq->mbuf[pi].num_wqebbs; 647 } 648 649 static void 650 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 651 { 652 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS; 653 struct mlx5e_tx_wqe *wqe; 654 u16 pi; 655 656 pi = sq->pc & sq->wq.sz_m1; 657 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 658 659 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 660 661 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 662 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 663 if (mlx5e_do_send_cqe(sq)) 664 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 665 else 666 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 667 668 /* Copy data for doorbell */ 669 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 670 671 sq->mbuf[pi].mbuf = NULL; 672 sq->mbuf[pi].num_bytes = 0; 673 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 674 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag); 675 676 sq->pc += sq->mbuf[pi].num_wqebbs; 677 } 678 679 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1) 680 681 static struct mbuf * 682 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start) 683 { 684 struct mbuf *mr, *top; 685 uint32_t offset; 686 uint32_t delta; 687 688 /* check format of incoming mbuf */ 689 if (mb->m_next == NULL || 690 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) { 691 top = NULL; 692 goto done; 693 } 694 695 /* get unmapped data offset */ 696 offset = mtod(mb->m_next, uintptr_t); 697 698 /* check if we don't need to re-transmit anything */ 699 if (offset == 0) { 700 top = SBTLS_MBUF_NO_DATA; 701 *pis_start = true; 702 goto done; 703 } 704 705 /* try to get a new packet header */ 706 top = m_gethdr(wait, MT_DATA); 707 if (top == NULL) 708 goto done; 709 710 mr = m_get(wait, MT_DATA); 711 if (mr == NULL) { 712 m_free(top); 713 top = NULL; 714 goto done; 715 } 716 717 top->m_next = mr; 718 719 mb_dupcl(mr, mb->m_next); 720 721 /* the beginning of the TLS record */ 722 mr->m_data = NULL; 723 724 /* setup packet header length */ 725 top->m_pkthdr.len = mr->m_len = offset; 726 top->m_len = 0; 727 728 /* check for partial re-transmit */ 729 delta = *ptcp_seq - tcp_old; 730 731 if (delta < offset) { 732 m_adj(top, offset - delta); 733 offset = delta; 734 735 /* continue where we left off */ 736 *pis_start = false; 737 } else { 738 *pis_start = true; 739 } 740 741 /* 742 * Rewind the TCP sequence number by the amount of data 743 * retransmitted: 744 */ 745 *ptcp_seq -= offset; 746 done: 747 return (top); 748 } 749 750 static int 751 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq) 752 { 753 754 for (; mb != NULL; mb = mb->m_next) { 755 if (!(mb->m_flags & M_EXTPG)) 756 continue; 757 *pseq = mb->m_epg_seqno; 758 return (1); 759 } 760 return (0); 761 } 762 763 int 764 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb) 765 { 766 struct mlx5e_tls_tag *ptls_tag; 767 struct m_snd_tag *ptag; 768 const struct tcphdr *th; 769 struct mbuf *mb = *ppmb; 770 u64 rcd_sn; 771 u32 header_size; 772 u32 mb_seq; 773 774 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0) 775 return (MLX5E_TLS_CONTINUE); 776 777 ptag = mb->m_pkthdr.snd_tag; 778 779 if ( 780 #ifdef RATELIMIT 781 ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT && 782 #endif 783 ptag->sw->type != IF_SND_TAG_TYPE_TLS) 784 return (MLX5E_TLS_CONTINUE); 785 786 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag); 787 788 header_size = mlx5e_get_full_header_size(mb, &th); 789 if (unlikely(header_size == 0 || th == NULL)) 790 return (MLX5E_TLS_FAILURE); 791 792 /* 793 * Send non-TLS TCP packets AS-IS: 794 */ 795 if (header_size == mb->m_pkthdr.len || 796 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) { 797 parg->tisn = 0; 798 parg->ihs = header_size; 799 return (MLX5E_TLS_CONTINUE); 800 } 801 802 mb_seq = ntohl(th->th_seq); 803 804 MLX5E_TLS_TAG_LOCK(ptls_tag); 805 switch (ptls_tag->state) { 806 case MLX5E_TLS_ST_INIT: 807 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 808 return (MLX5E_TLS_FAILURE); 809 case MLX5E_TLS_ST_SETUP: 810 ptls_tag->state = MLX5E_TLS_ST_TXRDY; 811 ptls_tag->expected_seq = ~mb_seq; /* force setup */ 812 default: 813 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 814 break; 815 } 816 817 if (unlikely(ptls_tag->expected_seq != mb_seq)) { 818 bool is_start; 819 struct mbuf *r_mb; 820 uint32_t tcp_seq = mb_seq; 821 822 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start); 823 if (r_mb == NULL) { 824 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1); 825 return (MLX5E_TLS_FAILURE); 826 } 827 828 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1); 829 830 /* check if this is the first fragment of a TLS record */ 831 if (is_start) { 832 /* setup TLS static parameters */ 833 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params, 834 param.initial_record_number, rcd_sn); 835 836 /* 837 * NOTE: The sendqueue should have enough room to 838 * carry both the static and the progress parameters 839 * when we get here! 840 */ 841 mlx5e_tls_send_static_parameters(sq, ptls_tag); 842 mlx5e_tls_send_progress_parameters(sq, ptls_tag); 843 844 if (r_mb == SBTLS_MBUF_NO_DATA) { 845 mlx5e_tls_send_nop(sq, ptls_tag); 846 ptls_tag->expected_seq = mb_seq; 847 return (MLX5E_TLS_LOOP); 848 } 849 } 850 851 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len); 852 853 /* setup transmit arguments */ 854 parg->tisn = ptls_tag->tisn; 855 parg->mst = &ptls_tag->tag; 856 857 /* try to send DUMP data */ 858 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) { 859 m_freem(r_mb); 860 ptls_tag->expected_seq = tcp_seq; 861 return (MLX5E_TLS_FAILURE); 862 } else { 863 ptls_tag->expected_seq = mb_seq; 864 return (MLX5E_TLS_LOOP); 865 } 866 } else { 867 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1); 868 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len); 869 } 870 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size; 871 872 parg->tisn = ptls_tag->tisn; 873 parg->ihs = header_size; 874 parg->mst = &ptls_tag->tag; 875 return (MLX5E_TLS_CONTINUE); 876 } 877 878 #else 879 880 int 881 mlx5e_tls_init(struct mlx5e_priv *priv) 882 { 883 884 return (0); 885 } 886 887 void 888 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 889 { 890 /* NOP */ 891 } 892 893 #endif /* KERN_TLS */ 894