1 /*- 2 * Copyright (c) 2019 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "opt_kern_tls.h" 29 30 #include "en.h" 31 32 #include <dev/mlx5/tls.h> 33 34 #include <linux/delay.h> 35 #include <sys/ktls.h> 36 #include <opencrypto/cryptodev.h> 37 38 #ifdef KERN_TLS 39 40 #ifdef RATELIMIT 41 static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify; 42 #endif 43 static if_snd_tag_query_t mlx5e_tls_snd_tag_query; 44 static if_snd_tag_free_t mlx5e_tls_snd_tag_free; 45 46 static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = { 47 .snd_tag_query = mlx5e_tls_snd_tag_query, 48 .snd_tag_free = mlx5e_tls_snd_tag_free, 49 .type = IF_SND_TAG_TYPE_TLS 50 }; 51 52 #ifdef RATELIMIT 53 static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = { 54 .snd_tag_modify = mlx5e_tls_rl_snd_tag_modify, 55 .snd_tag_query = mlx5e_tls_snd_tag_query, 56 .snd_tag_free = mlx5e_tls_snd_tag_free, 57 .type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT 58 }; 59 #endif 60 61 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS"); 62 63 /* software TLS context */ 64 struct mlx5_ifc_sw_tls_cntx_bits { 65 struct mlx5_ifc_tls_static_params_bits param; 66 struct mlx5_ifc_tls_progress_params_bits progress; 67 struct { 68 uint8_t key_data[8][0x20]; 69 uint8_t key_len[0x20]; 70 } key; 71 }; 72 73 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params)); 74 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc)); 75 76 static const char *mlx5e_tls_stats_desc[] = { 77 MLX5E_TLS_STATS(MLX5E_STATS_DESC) 78 }; 79 80 static void mlx5e_tls_work(struct work_struct *); 81 82 static int 83 mlx5e_tls_tag_zinit(void *mem, int size, int flags) 84 { 85 struct mlx5e_tls_tag *ptag = mem; 86 87 MPASS(size == sizeof(*ptag)); 88 89 memset(ptag, 0, sizeof(*ptag)); 90 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF); 91 INIT_WORK(&ptag->work, mlx5e_tls_work); 92 93 return (0); 94 } 95 96 static void 97 mlx5e_tls_tag_zfini(void *mem, int size) 98 { 99 struct mlx5e_tls_tag *ptag = mem; 100 struct mlx5e_priv *priv; 101 struct mlx5e_tls *ptls; 102 103 ptls = ptag->tls; 104 priv = container_of(ptls, struct mlx5e_priv, tls); 105 106 flush_work(&ptag->work); 107 108 if (ptag->tisn != 0) { 109 mlx5_tls_close_tis(priv->mdev, ptag->tisn); 110 atomic_add_32(&ptls->num_resources, -1U); 111 } 112 113 mtx_destroy(&ptag->mtx); 114 } 115 116 static void 117 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag) 118 { 119 120 /* reset some variables */ 121 ptag->state = MLX5E_TLS_ST_INIT; 122 ptag->dek_index = 0; 123 ptag->dek_index_ok = 0; 124 125 /* avoid leaking keys */ 126 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params)); 127 128 /* update number of TIS contexts */ 129 if (ptag->tisn == 0) 130 atomic_add_32(&ptag->tls->num_resources, -1U); 131 132 /* return tag to UMA */ 133 uma_zfree(ptag->tls->zone, ptag); 134 } 135 136 int 137 mlx5e_tls_init(struct mlx5e_priv *priv) 138 { 139 struct mlx5e_tls *ptls = &priv->tls; 140 struct sysctl_oid *node; 141 uint32_t x; 142 143 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0) 144 return (0); 145 146 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq"); 147 if (ptls->wq == NULL) 148 return (ENOMEM); 149 150 sysctl_ctx_init(&ptls->ctx); 151 152 snprintf(ptls->zname, sizeof(ptls->zname), 153 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev)); 154 155 ptls->zone = uma_zcreate(ptls->zname, sizeof(struct mlx5e_tls_tag), 156 NULL, NULL, mlx5e_tls_tag_zinit, mlx5e_tls_tag_zfini, UMA_ALIGN_CACHE, 0); 157 158 ptls->max_resources = 1U << MLX5_CAP_GEN(priv->mdev, log_max_dek); 159 160 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 161 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK); 162 163 ptls->init = 1; 164 165 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, 166 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, 167 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload"); 168 if (node == NULL) 169 return (0); 170 171 mlx5e_create_counter_stats(&ptls->ctx, 172 SYSCTL_CHILDREN(node), "stats", 173 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM, 174 ptls->stats.arg); 175 176 return (0); 177 } 178 179 void 180 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 181 { 182 struct mlx5e_tls *ptls = &priv->tls; 183 uint32_t x; 184 185 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0) 186 return; 187 188 ptls->init = 0; 189 flush_workqueue(ptls->wq); 190 sysctl_ctx_free(&ptls->ctx); 191 uma_zdestroy(ptls->zone); 192 destroy_workqueue(ptls->wq); 193 194 /* check if all resources are freed */ 195 MPASS(priv->tls.num_resources == 0); 196 197 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++) 198 counter_u64_free(ptls->stats.arg[x]); 199 } 200 201 static void 202 mlx5e_tls_work(struct work_struct *work) 203 { 204 struct mlx5e_tls_tag *ptag; 205 struct mlx5e_priv *priv; 206 int err; 207 208 ptag = container_of(work, struct mlx5e_tls_tag, work); 209 priv = container_of(ptag->tls, struct mlx5e_priv, tls); 210 211 switch (ptag->state) { 212 case MLX5E_TLS_ST_INIT: 213 /* try to open TIS, if not present */ 214 if (ptag->tisn == 0) { 215 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn, 216 priv->pdn, &ptag->tisn); 217 if (err) { 218 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 219 break; 220 } 221 } 222 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn); 223 224 /* try to allocate a DEK context ID */ 225 err = mlx5_encryption_key_create(priv->mdev, priv->pdn, 226 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data), 227 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len), 228 &ptag->dek_index); 229 if (err) { 230 MLX5E_TLS_STAT_INC(ptag, tx_error, 1); 231 break; 232 } 233 234 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index); 235 236 ptag->dek_index_ok = 1; 237 238 MLX5E_TLS_TAG_LOCK(ptag); 239 if (ptag->state == MLX5E_TLS_ST_INIT) 240 ptag->state = MLX5E_TLS_ST_SETUP; 241 MLX5E_TLS_TAG_UNLOCK(ptag); 242 break; 243 244 case MLX5E_TLS_ST_FREED: 245 /* wait for all refs to go away */ 246 while (ptag->refs != 0) 247 msleep(1); 248 249 /* try to destroy DEK context by ID */ 250 if (ptag->dek_index_ok) 251 err = mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index); 252 253 /* free tag */ 254 mlx5e_tls_tag_zfree(ptag); 255 break; 256 257 default: 258 break; 259 } 260 } 261 262 static int 263 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en) 264 { 265 266 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2); 267 if (en->tls_vminor == TLS_MINOR_VER_TWO) 268 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */ 269 else 270 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */ 271 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1); 272 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */ 273 274 /* copy the initial vector in place */ 275 switch (en->iv_len) { 276 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv): 277 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) + 278 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv): 279 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv), 280 en->iv, en->iv_len); 281 break; 282 default: 283 return (EINVAL); 284 } 285 286 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) { 287 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data), 288 en->cipher_key, en->cipher_key_len); 289 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len); 290 } else { 291 return (EINVAL); 292 } 293 return (0); 294 } 295 296 /* Verify zero default */ 297 CTASSERT(MLX5E_TLS_ST_INIT == 0); 298 299 int 300 mlx5e_tls_snd_tag_alloc(struct ifnet *ifp, 301 union if_snd_tag_alloc_params *params, 302 struct m_snd_tag **ppmt) 303 { 304 union if_snd_tag_alloc_params rl_params; 305 const struct if_snd_tag_sw *snd_tag_sw; 306 struct mlx5e_priv *priv; 307 struct mlx5e_tls_tag *ptag; 308 const struct tls_session_params *en; 309 int error; 310 311 priv = ifp->if_softc; 312 313 if (priv->gone != 0 || priv->tls.init == 0) 314 return (EOPNOTSUPP); 315 316 /* allocate new tag from zone, if any */ 317 ptag = uma_zalloc(priv->tls.zone, M_NOWAIT); 318 if (ptag == NULL) 319 return (ENOMEM); 320 321 /* sanity check default values */ 322 MPASS(ptag->state == MLX5E_TLS_ST_INIT); 323 MPASS(ptag->dek_index == 0); 324 MPASS(ptag->dek_index_ok == 0); 325 326 /* setup TLS tag */ 327 ptag->tls = &priv->tls; 328 329 /* check if there is no TIS context */ 330 if (ptag->tisn == 0) { 331 uint32_t value; 332 333 value = atomic_fetchadd_32(&priv->tls.num_resources, 1U); 334 335 /* check resource limits */ 336 if (value >= priv->tls.max_resources) { 337 error = ENOMEM; 338 goto failure; 339 } 340 } 341 342 en = ¶ms->tls.tls->params; 343 344 /* only TLS v1.2 and v1.3 is currently supported */ 345 if (en->tls_vmajor != TLS_MAJOR_VER_ONE || 346 (en->tls_vminor != TLS_MINOR_VER_TWO 347 #ifdef TLS_MINOR_VER_THREE 348 && en->tls_vminor != TLS_MINOR_VER_THREE 349 #endif 350 )) { 351 error = EPROTONOSUPPORT; 352 goto failure; 353 } 354 355 switch (en->cipher_algorithm) { 356 case CRYPTO_AES_NIST_GCM_16: 357 switch (en->cipher_key_len) { 358 case 128 / 8: 359 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 360 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) { 361 error = EPROTONOSUPPORT; 362 goto failure; 363 } 364 } else { 365 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) { 366 error = EPROTONOSUPPORT; 367 goto failure; 368 } 369 } 370 error = mlx5e_tls_set_params(ptag->crypto_params, en); 371 if (error) 372 goto failure; 373 break; 374 375 case 256 / 8: 376 if (en->tls_vminor == TLS_MINOR_VER_TWO) { 377 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) { 378 error = EPROTONOSUPPORT; 379 goto failure; 380 } 381 } else { 382 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) { 383 error = EPROTONOSUPPORT; 384 goto failure; 385 } 386 } 387 error = mlx5e_tls_set_params(ptag->crypto_params, en); 388 if (error) 389 goto failure; 390 break; 391 392 default: 393 error = EINVAL; 394 goto failure; 395 } 396 break; 397 default: 398 error = EPROTONOSUPPORT; 399 goto failure; 400 } 401 402 memset(&rl_params, 0, sizeof(rl_params)); 403 rl_params.hdr = params->hdr; 404 switch (params->hdr.type) { 405 #ifdef RATELIMIT 406 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT: 407 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT; 408 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 409 snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw; 410 break; 411 #endif 412 case IF_SND_TAG_TYPE_TLS: 413 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED; 414 snd_tag_sw = &mlx5e_tls_snd_tag_sw; 415 break; 416 default: 417 error = EOPNOTSUPP; 418 goto failure; 419 } 420 421 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag); 422 if (error) 423 goto failure; 424 425 /* store pointer to mbuf tag */ 426 MPASS(ptag->tag.refcount == 0); 427 m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw); 428 *ppmt = &ptag->tag; 429 430 queue_work(priv->tls.wq, &ptag->work); 431 flush_work(&ptag->work); 432 433 return (0); 434 435 failure: 436 mlx5e_tls_tag_zfree(ptag); 437 return (error); 438 } 439 440 #ifdef RATELIMIT 441 static int 442 mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params) 443 { 444 union if_snd_tag_modify_params rl_params; 445 struct mlx5e_tls_tag *ptag = 446 container_of(pmt, struct mlx5e_tls_tag, tag); 447 int error; 448 449 memset(&rl_params, 0, sizeof(rl_params)); 450 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate; 451 error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params); 452 return (error); 453 } 454 #endif 455 456 static int 457 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params) 458 { 459 struct mlx5e_tls_tag *ptag = 460 container_of(pmt, struct mlx5e_tls_tag, tag); 461 462 return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params)); 463 } 464 465 static void 466 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt) 467 { 468 struct mlx5e_tls_tag *ptag = 469 container_of(pmt, struct mlx5e_tls_tag, tag); 470 struct mlx5e_priv *priv; 471 472 m_snd_tag_rele(ptag->rl_tag); 473 474 MLX5E_TLS_TAG_LOCK(ptag); 475 ptag->state = MLX5E_TLS_ST_FREED; 476 MLX5E_TLS_TAG_UNLOCK(ptag); 477 478 priv = ptag->tag.ifp->if_softc; 479 queue_work(priv->tls.wq, &ptag->work); 480 } 481 482 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0); 483 484 static void 485 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 486 { 487 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) + 488 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS); 489 struct mlx5e_tx_umr_wqe *wqe; 490 u16 pi; 491 492 pi = sq->pc & sq->wq.sz_m1; 493 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 494 495 memset(wqe, 0, sizeof(*wqe)); 496 497 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 498 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24)); 499 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 500 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8); 501 502 if (mlx5e_do_send_cqe(sq)) 503 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 504 else 505 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 506 507 /* fill out UMR control segment */ 508 wqe->umr.flags = 0x80; /* inline data */ 509 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16); 510 511 /* copy in the static crypto parameters */ 512 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param), 513 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param)); 514 515 /* copy data for doorbell */ 516 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 517 518 sq->mbuf[pi].mbuf = NULL; 519 sq->mbuf[pi].num_bytes = 0; 520 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 521 sq->mbuf[pi].p_refcount = &ptag->refs; 522 atomic_add_int(&ptag->refs, 1); 523 sq->pc += sq->mbuf[pi].num_wqebbs; 524 } 525 526 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) == 527 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv)); 528 529 static void 530 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 531 { 532 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe), 533 MLX5_SEND_WQE_DS); 534 struct mlx5e_tx_psv_wqe *wqe; 535 u16 pi; 536 537 pi = sq->pc & sq->wq.sz_m1; 538 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 539 540 memset(wqe, 0, sizeof(*wqe)); 541 542 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | 543 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24)); 544 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 545 546 if (mlx5e_do_send_cqe(sq)) 547 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 548 549 /* copy in the PSV control segment */ 550 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress), 551 sizeof(wqe->psv)); 552 553 /* copy data for doorbell */ 554 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 555 556 sq->mbuf[pi].mbuf = NULL; 557 sq->mbuf[pi].num_bytes = 0; 558 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 559 sq->mbuf[pi].p_refcount = &ptag->refs; 560 atomic_add_int(&ptag->refs, 1); 561 sq->pc += sq->mbuf[pi].num_wqebbs; 562 } 563 564 static void 565 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag) 566 { 567 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS; 568 struct mlx5e_tx_wqe *wqe; 569 u16 pi; 570 571 pi = sq->pc & sq->wq.sz_m1; 572 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 573 574 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 575 576 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 577 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 578 if (mlx5e_do_send_cqe(sq)) 579 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL; 580 else 581 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL; 582 583 /* Copy data for doorbell */ 584 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 585 586 sq->mbuf[pi].mbuf = NULL; 587 sq->mbuf[pi].num_bytes = 0; 588 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 589 sq->mbuf[pi].p_refcount = &ptag->refs; 590 atomic_add_int(&ptag->refs, 1); 591 sq->pc += sq->mbuf[pi].num_wqebbs; 592 } 593 594 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1) 595 596 static struct mbuf * 597 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start) 598 { 599 struct mbuf *mr, *top; 600 uint32_t offset; 601 uint32_t delta; 602 603 /* check format of incoming mbuf */ 604 if (mb->m_next == NULL || 605 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) { 606 top = NULL; 607 goto done; 608 } 609 610 /* get unmapped data offset */ 611 offset = mtod(mb->m_next, uintptr_t); 612 613 /* check if we don't need to re-transmit anything */ 614 if (offset == 0) { 615 top = SBTLS_MBUF_NO_DATA; 616 *pis_start = true; 617 goto done; 618 } 619 620 /* try to get a new packet header */ 621 top = m_gethdr(wait, MT_DATA); 622 if (top == NULL) 623 goto done; 624 625 mr = m_get(wait, MT_DATA); 626 if (mr == NULL) { 627 m_free(top); 628 top = NULL; 629 goto done; 630 } 631 632 top->m_next = mr; 633 634 mb_dupcl(mr, mb->m_next); 635 636 /* the beginning of the TLS record */ 637 mr->m_data = NULL; 638 639 /* setup packet header length */ 640 top->m_pkthdr.len = mr->m_len = offset; 641 top->m_len = 0; 642 643 /* check for partial re-transmit */ 644 delta = *ptcp_seq - tcp_old; 645 646 if (delta < offset) { 647 m_adj(top, offset - delta); 648 offset = delta; 649 650 /* continue where we left off */ 651 *pis_start = false; 652 } else { 653 *pis_start = true; 654 } 655 656 /* 657 * Rewind the TCP sequence number by the amount of data 658 * retransmitted: 659 */ 660 *ptcp_seq -= offset; 661 done: 662 return (top); 663 } 664 665 static int 666 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq) 667 { 668 669 for (; mb != NULL; mb = mb->m_next) { 670 if (!(mb->m_flags & M_EXTPG)) 671 continue; 672 *pseq = mb->m_epg_seqno; 673 return (1); 674 } 675 return (0); 676 } 677 678 int 679 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb) 680 { 681 struct mlx5e_tls_tag *ptls_tag; 682 struct m_snd_tag *ptag; 683 const struct tcphdr *th; 684 struct mbuf *mb = *ppmb; 685 u64 rcd_sn; 686 u32 header_size; 687 u32 mb_seq; 688 689 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0) 690 return (MLX5E_TLS_CONTINUE); 691 692 ptag = mb->m_pkthdr.snd_tag; 693 694 if ( 695 #ifdef RATELIMIT 696 ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT && 697 #endif 698 ptag->sw->type != IF_SND_TAG_TYPE_TLS) 699 return (MLX5E_TLS_CONTINUE); 700 701 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag); 702 703 header_size = mlx5e_get_full_header_size(mb, &th); 704 if (unlikely(header_size == 0 || th == NULL)) 705 return (MLX5E_TLS_FAILURE); 706 707 /* 708 * Send non-TLS TCP packets AS-IS: 709 */ 710 if (header_size == mb->m_pkthdr.len || 711 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) { 712 parg->tisn = 0; 713 parg->ihs = header_size; 714 return (MLX5E_TLS_CONTINUE); 715 } 716 717 mb_seq = ntohl(th->th_seq); 718 719 MLX5E_TLS_TAG_LOCK(ptls_tag); 720 switch (ptls_tag->state) { 721 case MLX5E_TLS_ST_INIT: 722 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 723 return (MLX5E_TLS_FAILURE); 724 case MLX5E_TLS_ST_SETUP: 725 ptls_tag->state = MLX5E_TLS_ST_TXRDY; 726 ptls_tag->expected_seq = ~mb_seq; /* force setup */ 727 default: 728 MLX5E_TLS_TAG_UNLOCK(ptls_tag); 729 break; 730 } 731 732 if (unlikely(ptls_tag->expected_seq != mb_seq)) { 733 bool is_start; 734 struct mbuf *r_mb; 735 uint32_t tcp_seq = mb_seq; 736 737 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start); 738 if (r_mb == NULL) { 739 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1); 740 return (MLX5E_TLS_FAILURE); 741 } 742 743 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1); 744 745 /* check if this is the first fragment of a TLS record */ 746 if (is_start) { 747 /* setup TLS static parameters */ 748 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params, 749 param.initial_record_number, rcd_sn); 750 751 /* 752 * NOTE: The sendqueue should have enough room to 753 * carry both the static and the progress parameters 754 * when we get here! 755 */ 756 mlx5e_tls_send_static_parameters(sq, ptls_tag); 757 mlx5e_tls_send_progress_parameters(sq, ptls_tag); 758 759 if (r_mb == SBTLS_MBUF_NO_DATA) { 760 mlx5e_tls_send_nop(sq, ptls_tag); 761 ptls_tag->expected_seq = mb_seq; 762 return (MLX5E_TLS_LOOP); 763 } 764 } 765 766 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len); 767 768 /* setup transmit arguments */ 769 parg->tisn = ptls_tag->tisn; 770 parg->pref = &ptls_tag->refs; 771 772 /* try to send DUMP data */ 773 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) { 774 m_freem(r_mb); 775 ptls_tag->expected_seq = tcp_seq; 776 return (MLX5E_TLS_FAILURE); 777 } else { 778 ptls_tag->expected_seq = mb_seq; 779 return (MLX5E_TLS_LOOP); 780 } 781 } else { 782 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1); 783 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len); 784 } 785 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size; 786 787 parg->tisn = ptls_tag->tisn; 788 parg->ihs = header_size; 789 parg->pref = &ptls_tag->refs; 790 return (MLX5E_TLS_CONTINUE); 791 } 792 793 #else 794 795 int 796 mlx5e_tls_init(struct mlx5e_priv *priv) 797 { 798 799 return (0); 800 } 801 802 void 803 mlx5e_tls_cleanup(struct mlx5e_priv *priv) 804 { 805 /* NOP */ 806 } 807 808 #endif /* KERN_TLS */ 809