1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/counter.h> 34 #include <sys/endian.h> 35 #include <sys/kernel.h> 36 #include <sys/ktls.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <vm/vm.h> 45 #include <vm/pmap.h> 46 #include <vm/vm_param.h> 47 #include <opencrypto/cryptodev.h> 48 #include <opencrypto/ktls.h> 49 50 struct ktls_ocf_session { 51 crypto_session_t sid; 52 crypto_session_t mac_sid; 53 struct mtx lock; 54 int mac_len; 55 bool implicit_iv; 56 57 /* Only used for TLS 1.0 with the implicit IV. */ 58 #ifdef INVARIANTS 59 bool in_progress; 60 uint64_t next_seqno; 61 #endif 62 char iv[AES_BLOCK_LEN]; 63 }; 64 65 struct ocf_operation { 66 struct ktls_ocf_session *os; 67 bool done; 68 }; 69 70 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 71 72 SYSCTL_DECL(_kern_ipc_tls); 73 SYSCTL_DECL(_kern_ipc_tls_stats); 74 75 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 76 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 77 "Kernel TLS offload via OCF stats"); 78 79 static COUNTER_U64_DEFINE_EARLY(ocf_tls10_cbc_crypts); 80 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls10_cbc_crypts, 81 CTLFLAG_RD, &ocf_tls10_cbc_crypts, 82 "Total number of OCF TLS 1.0 CBC encryption operations"); 83 84 static COUNTER_U64_DEFINE_EARLY(ocf_tls11_cbc_crypts); 85 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_crypts, 86 CTLFLAG_RD, &ocf_tls11_cbc_crypts, 87 "Total number of OCF TLS 1.1/1.2 CBC encryption operations"); 88 89 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_crypts); 90 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 91 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 92 "Total number of OCF TLS 1.2 GCM encryption operations"); 93 94 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_crypts); 95 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_crypts, 96 CTLFLAG_RD, &ocf_tls12_chacha20_crypts, 97 "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); 98 99 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts); 100 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 101 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 102 "Total number of OCF TLS 1.3 GCM encryption operations"); 103 104 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_crypts); 105 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_crypts, 106 CTLFLAG_RD, &ocf_tls13_chacha20_crypts, 107 "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); 108 109 static COUNTER_U64_DEFINE_EARLY(ocf_inplace); 110 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 111 CTLFLAG_RD, &ocf_inplace, 112 "Total number of OCF in-place operations"); 113 114 static COUNTER_U64_DEFINE_EARLY(ocf_separate_output); 115 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 116 CTLFLAG_RD, &ocf_separate_output, 117 "Total number of OCF operations with a separate output buffer"); 118 119 static COUNTER_U64_DEFINE_EARLY(ocf_retries); 120 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 121 &ocf_retries, 122 "Number of OCF encryption operation retries"); 123 124 static int 125 ktls_ocf_callback_sync(struct cryptop *crp __unused) 126 { 127 return (0); 128 } 129 130 static int 131 ktls_ocf_callback_async(struct cryptop *crp) 132 { 133 struct ocf_operation *oo; 134 135 oo = crp->crp_opaque; 136 mtx_lock(&oo->os->lock); 137 oo->done = true; 138 mtx_unlock(&oo->os->lock); 139 wakeup(oo); 140 return (0); 141 } 142 143 static int 144 ktls_ocf_dispatch(struct ktls_ocf_session *os, struct cryptop *crp) 145 { 146 struct ocf_operation oo; 147 int error; 148 bool async; 149 150 oo.os = os; 151 oo.done = false; 152 153 crp->crp_opaque = &oo; 154 for (;;) { 155 async = !CRYPTO_SESS_SYNC(crp->crp_session); 156 crp->crp_callback = async ? ktls_ocf_callback_async : 157 ktls_ocf_callback_sync; 158 159 error = crypto_dispatch(crp); 160 if (error) 161 break; 162 if (async) { 163 mtx_lock(&os->lock); 164 while (!oo.done) 165 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0); 166 mtx_unlock(&os->lock); 167 } 168 169 if (crp->crp_etype != EAGAIN) { 170 error = crp->crp_etype; 171 break; 172 } 173 174 crp->crp_etype = 0; 175 crp->crp_flags &= ~CRYPTO_F_DONE; 176 oo.done = false; 177 counter_u64_add(ocf_retries, 1); 178 } 179 return (error); 180 } 181 182 static int 183 ktls_ocf_dispatch_async_cb(struct cryptop *crp) 184 { 185 struct ktls_ocf_encrypt_state *state; 186 int error; 187 188 state = crp->crp_opaque; 189 if (crp->crp_etype == EAGAIN) { 190 crp->crp_etype = 0; 191 crp->crp_flags &= ~CRYPTO_F_DONE; 192 counter_u64_add(ocf_retries, 1); 193 error = crypto_dispatch(crp); 194 if (error != 0) { 195 crypto_destroyreq(crp); 196 ktls_encrypt_cb(state, error); 197 } 198 return (0); 199 } 200 201 error = crp->crp_etype; 202 crypto_destroyreq(crp); 203 ktls_encrypt_cb(state, error); 204 return (0); 205 } 206 207 static int 208 ktls_ocf_dispatch_async(struct ktls_ocf_encrypt_state *state, 209 struct cryptop *crp) 210 { 211 int error; 212 213 crp->crp_opaque = state; 214 crp->crp_callback = ktls_ocf_dispatch_async_cb; 215 error = crypto_dispatch(crp); 216 if (error != 0) 217 crypto_destroyreq(crp); 218 return (error); 219 } 220 221 static int 222 ktls_ocf_tls_cbc_encrypt(struct ktls_ocf_encrypt_state *state, 223 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 224 int outiovcnt) 225 { 226 const struct tls_record_layer *hdr; 227 struct uio *uio; 228 struct tls_mac_data *ad; 229 struct cryptop *crp; 230 struct ktls_ocf_session *os; 231 struct iovec iov[m->m_epg_npgs + 2]; 232 u_int pgoff; 233 int i, error; 234 uint16_t tls_comp_len; 235 uint8_t pad; 236 237 MPASS(outiovcnt + 1 <= nitems(iov)); 238 239 os = tls->ocf_session; 240 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 241 crp = &state->crp; 242 uio = &state->uio; 243 MPASS(tls->sync_dispatch); 244 245 #ifdef INVARIANTS 246 if (os->implicit_iv) { 247 mtx_lock(&os->lock); 248 KASSERT(!os->in_progress, 249 ("concurrent implicit IV encryptions")); 250 if (os->next_seqno != m->m_epg_seqno) { 251 printf("KTLS CBC: TLS records out of order. " 252 "Expected %ju, got %ju\n", 253 (uintmax_t)os->next_seqno, 254 (uintmax_t)m->m_epg_seqno); 255 mtx_unlock(&os->lock); 256 return (EINVAL); 257 } 258 os->in_progress = true; 259 mtx_unlock(&os->lock); 260 } 261 #endif 262 263 /* Payload length. */ 264 tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen); 265 266 /* Initialize the AAD. */ 267 ad = &state->mac; 268 ad->seq = htobe64(m->m_epg_seqno); 269 ad->type = hdr->tls_type; 270 ad->tls_vmajor = hdr->tls_vmajor; 271 ad->tls_vminor = hdr->tls_vminor; 272 ad->tls_length = htons(tls_comp_len); 273 274 /* First, compute the MAC. */ 275 iov[0].iov_base = ad; 276 iov[0].iov_len = sizeof(*ad); 277 pgoff = m->m_epg_1st_off; 278 for (i = 0; i < m->m_epg_npgs; i++, pgoff = 0) { 279 iov[i + 1].iov_base = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + 280 pgoff); 281 iov[i + 1].iov_len = m_epg_pagelen(m, i, pgoff); 282 } 283 iov[m->m_epg_npgs + 1].iov_base = m->m_epg_trail; 284 iov[m->m_epg_npgs + 1].iov_len = os->mac_len; 285 uio->uio_iov = iov; 286 uio->uio_iovcnt = m->m_epg_npgs + 2; 287 uio->uio_offset = 0; 288 uio->uio_segflg = UIO_SYSSPACE; 289 uio->uio_td = curthread; 290 uio->uio_resid = sizeof(*ad) + tls_comp_len + os->mac_len; 291 292 crypto_initreq(crp, os->mac_sid); 293 crp->crp_payload_start = 0; 294 crp->crp_payload_length = sizeof(*ad) + tls_comp_len; 295 crp->crp_digest_start = crp->crp_payload_length; 296 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 297 crp->crp_flags = CRYPTO_F_CBIMM; 298 crypto_use_uio(crp, uio); 299 error = ktls_ocf_dispatch(os, crp); 300 301 crypto_destroyreq(crp); 302 if (error) { 303 #ifdef INVARIANTS 304 if (os->implicit_iv) { 305 mtx_lock(&os->lock); 306 os->in_progress = false; 307 mtx_unlock(&os->lock); 308 } 309 #endif 310 return (error); 311 } 312 313 /* Second, add the padding. */ 314 pad = m->m_epg_trllen - os->mac_len - 1; 315 for (i = 0; i < pad + 1; i++) 316 m->m_epg_trail[os->mac_len + i] = pad; 317 318 /* Finally, encrypt the record. */ 319 crypto_initreq(crp, os->sid); 320 crp->crp_payload_start = m->m_epg_hdrlen; 321 crp->crp_payload_length = tls_comp_len + m->m_epg_trllen; 322 KASSERT(crp->crp_payload_length % AES_BLOCK_LEN == 0, 323 ("invalid encryption size")); 324 crypto_use_single_mbuf(crp, m); 325 crp->crp_op = CRYPTO_OP_ENCRYPT; 326 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 327 if (os->implicit_iv) 328 memcpy(crp->crp_iv, os->iv, AES_BLOCK_LEN); 329 else 330 memcpy(crp->crp_iv, hdr + 1, AES_BLOCK_LEN); 331 332 if (outiov != NULL) { 333 uio->uio_iov = outiov; 334 uio->uio_iovcnt = outiovcnt; 335 uio->uio_offset = 0; 336 uio->uio_segflg = UIO_SYSSPACE; 337 uio->uio_td = curthread; 338 uio->uio_resid = crp->crp_payload_length; 339 crypto_use_output_uio(crp, uio); 340 } 341 342 if (os->implicit_iv) 343 counter_u64_add(ocf_tls10_cbc_crypts, 1); 344 else 345 counter_u64_add(ocf_tls11_cbc_crypts, 1); 346 if (outiov != NULL) 347 counter_u64_add(ocf_separate_output, 1); 348 else 349 counter_u64_add(ocf_inplace, 1); 350 error = ktls_ocf_dispatch(os, crp); 351 352 crypto_destroyreq(crp); 353 354 if (os->implicit_iv) { 355 KASSERT(os->mac_len + pad + 1 >= AES_BLOCK_LEN, 356 ("trailer too short to read IV")); 357 memcpy(os->iv, m->m_epg_trail + m->m_epg_trllen - AES_BLOCK_LEN, 358 AES_BLOCK_LEN); 359 #ifdef INVARIANTS 360 mtx_lock(&os->lock); 361 os->next_seqno = m->m_epg_seqno + 1; 362 os->in_progress = false; 363 mtx_unlock(&os->lock); 364 #endif 365 } 366 return (error); 367 } 368 369 static int 370 ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state, 371 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 372 int outiovcnt) 373 { 374 const struct tls_record_layer *hdr; 375 struct uio *uio; 376 struct tls_aead_data *ad; 377 struct cryptop *crp; 378 struct ktls_ocf_session *os; 379 int error; 380 uint16_t tls_comp_len; 381 382 os = tls->ocf_session; 383 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 384 crp = &state->crp; 385 uio = &state->uio; 386 387 crypto_initreq(crp, os->sid); 388 389 /* Setup the IV. */ 390 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 391 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 392 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, 393 sizeof(uint64_t)); 394 } else { 395 /* 396 * Chacha20-Poly1305 constructs the IV for TLS 1.2 397 * identically to constructing the IV for AEAD in TLS 398 * 1.3. 399 */ 400 memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len); 401 *(uint64_t *)(crp->crp_iv + 4) ^= htobe64(m->m_epg_seqno); 402 } 403 404 /* Setup the AAD. */ 405 ad = &state->aead; 406 tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen); 407 ad->seq = htobe64(m->m_epg_seqno); 408 ad->type = hdr->tls_type; 409 ad->tls_vmajor = hdr->tls_vmajor; 410 ad->tls_vminor = hdr->tls_vminor; 411 ad->tls_length = htons(tls_comp_len); 412 crp->crp_aad = ad; 413 crp->crp_aad_length = sizeof(*ad); 414 415 /* Set fields for input payload. */ 416 crypto_use_single_mbuf(crp, m); 417 crp->crp_payload_start = m->m_epg_hdrlen; 418 crp->crp_payload_length = tls_comp_len; 419 420 if (outiov != NULL) { 421 crp->crp_digest_start = crp->crp_payload_length; 422 423 uio->uio_iov = outiov; 424 uio->uio_iovcnt = outiovcnt; 425 uio->uio_offset = 0; 426 uio->uio_segflg = UIO_SYSSPACE; 427 uio->uio_td = curthread; 428 uio->uio_resid = crp->crp_payload_length + tls->params.tls_tlen; 429 crypto_use_output_uio(crp, uio); 430 } else 431 crp->crp_digest_start = crp->crp_payload_start + 432 crp->crp_payload_length; 433 434 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 435 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 436 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 437 counter_u64_add(ocf_tls12_gcm_crypts, 1); 438 else 439 counter_u64_add(ocf_tls12_chacha20_crypts, 1); 440 if (outiov != NULL) 441 counter_u64_add(ocf_separate_output, 1); 442 else 443 counter_u64_add(ocf_inplace, 1); 444 if (tls->sync_dispatch) { 445 error = ktls_ocf_dispatch(os, crp); 446 crypto_destroyreq(crp); 447 } else 448 error = ktls_ocf_dispatch_async(state, crp); 449 return (error); 450 } 451 452 static int 453 ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, 454 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, 455 int *trailer_len) 456 { 457 struct tls_aead_data ad; 458 struct cryptop crp; 459 struct ktls_ocf_session *os; 460 int error; 461 uint16_t tls_comp_len; 462 463 os = tls->ocf_session; 464 465 crypto_initreq(&crp, os->sid); 466 467 /* Setup the IV. */ 468 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 469 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 470 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, 471 sizeof(uint64_t)); 472 } else { 473 /* 474 * Chacha20-Poly1305 constructs the IV for TLS 1.2 475 * identically to constructing the IV for AEAD in TLS 476 * 1.3. 477 */ 478 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); 479 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); 480 } 481 482 /* Setup the AAD. */ 483 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 484 tls_comp_len = ntohs(hdr->tls_length) - 485 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 486 else 487 tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN; 488 ad.seq = htobe64(seqno); 489 ad.type = hdr->tls_type; 490 ad.tls_vmajor = hdr->tls_vmajor; 491 ad.tls_vminor = hdr->tls_vminor; 492 ad.tls_length = htons(tls_comp_len); 493 crp.crp_aad = &ad; 494 crp.crp_aad_length = sizeof(ad); 495 496 crp.crp_payload_start = tls->params.tls_hlen; 497 crp.crp_payload_length = tls_comp_len; 498 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; 499 500 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; 501 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 502 crypto_use_mbuf(&crp, m); 503 504 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 505 counter_u64_add(ocf_tls12_gcm_crypts, 1); 506 else 507 counter_u64_add(ocf_tls12_chacha20_crypts, 1); 508 error = ktls_ocf_dispatch(os, &crp); 509 510 crypto_destroyreq(&crp); 511 *trailer_len = tls->params.tls_tlen; 512 return (error); 513 } 514 515 static int 516 ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, 517 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 518 int outiovcnt) 519 { 520 const struct tls_record_layer *hdr; 521 struct uio *uio; 522 struct tls_aead_data_13 *ad; 523 struct cryptop *crp; 524 struct ktls_ocf_session *os; 525 char nonce[12]; 526 int error; 527 528 os = tls->ocf_session; 529 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 530 crp = &state->crp; 531 uio = &state->uio; 532 533 crypto_initreq(crp, os->sid); 534 535 /* Setup the nonce. */ 536 memcpy(nonce, tls->params.iv, tls->params.iv_len); 537 *(uint64_t *)(nonce + 4) ^= htobe64(m->m_epg_seqno); 538 539 /* Setup the AAD. */ 540 ad = &state->aead13; 541 ad->type = hdr->tls_type; 542 ad->tls_vmajor = hdr->tls_vmajor; 543 ad->tls_vminor = hdr->tls_vminor; 544 ad->tls_length = hdr->tls_length; 545 crp->crp_aad = ad; 546 crp->crp_aad_length = sizeof(*ad); 547 548 /* Set fields for input payload. */ 549 crypto_use_single_mbuf(crp, m); 550 crp->crp_payload_start = m->m_epg_hdrlen; 551 crp->crp_payload_length = m->m_len - 552 (m->m_epg_hdrlen + m->m_epg_trllen); 553 554 /* Store the record type as the first byte of the trailer. */ 555 m->m_epg_trail[0] = m->m_epg_record_type; 556 crp->crp_payload_length++; 557 558 if (outiov != NULL) { 559 crp->crp_digest_start = crp->crp_payload_length; 560 561 uio->uio_iov = outiov; 562 uio->uio_iovcnt = outiovcnt; 563 uio->uio_offset = 0; 564 uio->uio_segflg = UIO_SYSSPACE; 565 uio->uio_td = curthread; 566 uio->uio_resid = m->m_len - m->m_epg_hdrlen; 567 crypto_use_output_uio(crp, uio); 568 } else 569 crp->crp_digest_start = crp->crp_payload_start + 570 crp->crp_payload_length; 571 572 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 573 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 574 575 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 576 577 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 578 counter_u64_add(ocf_tls13_gcm_crypts, 1); 579 else 580 counter_u64_add(ocf_tls13_chacha20_crypts, 1); 581 if (outiov != NULL) 582 counter_u64_add(ocf_separate_output, 1); 583 else 584 counter_u64_add(ocf_inplace, 1); 585 if (tls->sync_dispatch) { 586 error = ktls_ocf_dispatch(os, crp); 587 crypto_destroyreq(crp); 588 } else 589 error = ktls_ocf_dispatch_async(state, crp); 590 return (error); 591 } 592 593 void 594 ktls_ocf_free(struct ktls_session *tls) 595 { 596 struct ktls_ocf_session *os; 597 598 os = tls->ocf_session; 599 crypto_freesession(os->sid); 600 mtx_destroy(&os->lock); 601 zfree(os, M_KTLS_OCF); 602 } 603 604 int 605 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) 606 { 607 struct crypto_session_params csp, mac_csp; 608 struct ktls_ocf_session *os; 609 int error, mac_len; 610 611 memset(&csp, 0, sizeof(csp)); 612 memset(&mac_csp, 0, sizeof(mac_csp)); 613 mac_csp.csp_mode = CSP_MODE_NONE; 614 mac_len = 0; 615 616 switch (tls->params.cipher_algorithm) { 617 case CRYPTO_AES_NIST_GCM_16: 618 switch (tls->params.cipher_key_len) { 619 case 128 / 8: 620 case 256 / 8: 621 break; 622 default: 623 return (EINVAL); 624 } 625 626 /* Only TLS 1.2 and 1.3 are supported. */ 627 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 628 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 629 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 630 return (EPROTONOSUPPORT); 631 632 /* TLS 1.3 is not yet supported for receive. */ 633 if (direction == KTLS_RX && 634 tls->params.tls_vminor == TLS_MINOR_VER_THREE) 635 return (EPROTONOSUPPORT); 636 637 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 638 csp.csp_mode = CSP_MODE_AEAD; 639 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 640 csp.csp_cipher_key = tls->params.cipher_key; 641 csp.csp_cipher_klen = tls->params.cipher_key_len; 642 csp.csp_ivlen = AES_GCM_IV_LEN; 643 break; 644 case CRYPTO_AES_CBC: 645 switch (tls->params.cipher_key_len) { 646 case 128 / 8: 647 case 256 / 8: 648 break; 649 default: 650 return (EINVAL); 651 } 652 653 switch (tls->params.auth_algorithm) { 654 case CRYPTO_SHA1_HMAC: 655 mac_len = SHA1_HASH_LEN; 656 break; 657 case CRYPTO_SHA2_256_HMAC: 658 mac_len = SHA2_256_HASH_LEN; 659 break; 660 case CRYPTO_SHA2_384_HMAC: 661 mac_len = SHA2_384_HASH_LEN; 662 break; 663 default: 664 return (EINVAL); 665 } 666 667 /* Only TLS 1.0-1.2 are supported. */ 668 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 669 tls->params.tls_vminor < TLS_MINOR_VER_ZERO || 670 tls->params.tls_vminor > TLS_MINOR_VER_TWO) 671 return (EPROTONOSUPPORT); 672 673 /* AES-CBC is not supported for receive. */ 674 if (direction == KTLS_RX) 675 return (EPROTONOSUPPORT); 676 677 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 678 csp.csp_mode = CSP_MODE_CIPHER; 679 csp.csp_cipher_alg = CRYPTO_AES_CBC; 680 csp.csp_cipher_key = tls->params.cipher_key; 681 csp.csp_cipher_klen = tls->params.cipher_key_len; 682 csp.csp_ivlen = AES_BLOCK_LEN; 683 684 mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 685 mac_csp.csp_mode = CSP_MODE_DIGEST; 686 mac_csp.csp_auth_alg = tls->params.auth_algorithm; 687 mac_csp.csp_auth_key = tls->params.auth_key; 688 mac_csp.csp_auth_klen = tls->params.auth_key_len; 689 break; 690 case CRYPTO_CHACHA20_POLY1305: 691 switch (tls->params.cipher_key_len) { 692 case 256 / 8: 693 break; 694 default: 695 return (EINVAL); 696 } 697 698 /* Only TLS 1.2 and 1.3 are supported. */ 699 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 700 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 701 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 702 return (EPROTONOSUPPORT); 703 704 /* TLS 1.3 is not yet supported for receive. */ 705 if (direction == KTLS_RX && 706 tls->params.tls_vminor == TLS_MINOR_VER_THREE) 707 return (EPROTONOSUPPORT); 708 709 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 710 csp.csp_mode = CSP_MODE_AEAD; 711 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 712 csp.csp_cipher_key = tls->params.cipher_key; 713 csp.csp_cipher_klen = tls->params.cipher_key_len; 714 csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN; 715 break; 716 default: 717 return (EPROTONOSUPPORT); 718 } 719 720 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 721 if (os == NULL) 722 return (ENOMEM); 723 724 error = crypto_newsession(&os->sid, &csp, 725 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 726 if (error) { 727 free(os, M_KTLS_OCF); 728 return (error); 729 } 730 731 if (mac_csp.csp_mode != CSP_MODE_NONE) { 732 error = crypto_newsession(&os->mac_sid, &mac_csp, 733 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 734 if (error) { 735 crypto_freesession(os->sid); 736 free(os, M_KTLS_OCF); 737 return (error); 738 } 739 os->mac_len = mac_len; 740 } 741 742 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 743 tls->ocf_session = os; 744 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 || 745 tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { 746 if (direction == KTLS_TX) { 747 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 748 tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; 749 else 750 tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; 751 } else { 752 tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; 753 } 754 } else { 755 tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; 756 if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) { 757 os->implicit_iv = true; 758 memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN); 759 #ifdef INVARIANTS 760 os->next_seqno = tls->next_seqno; 761 #endif 762 } 763 } 764 765 /* 766 * AES-CBC is always synchronous currently. Asynchronous 767 * operation would require multiple callbacks and an additional 768 * iovec array in ktls_ocf_encrypt_state. 769 */ 770 tls->sync_dispatch = CRYPTO_SESS_SYNC(os->sid) || 771 tls->params.cipher_algorithm == CRYPTO_AES_CBC; 772 return (0); 773 } 774