1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/counter.h> 34 #include <sys/endian.h> 35 #include <sys/kernel.h> 36 #include <sys/ktls.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/mbuf.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <vm/vm.h> 45 #include <vm/pmap.h> 46 #include <vm/vm_param.h> 47 #include <opencrypto/cryptodev.h> 48 #include <opencrypto/ktls.h> 49 50 struct ktls_ocf_session { 51 crypto_session_t sid; 52 crypto_session_t mac_sid; 53 struct mtx lock; 54 int mac_len; 55 bool implicit_iv; 56 57 /* Only used for TLS 1.0 with the implicit IV. */ 58 #ifdef INVARIANTS 59 bool in_progress; 60 uint64_t next_seqno; 61 #endif 62 char iv[AES_BLOCK_LEN]; 63 }; 64 65 struct ocf_operation { 66 struct ktls_ocf_session *os; 67 bool done; 68 }; 69 70 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 71 72 SYSCTL_DECL(_kern_ipc_tls); 73 SYSCTL_DECL(_kern_ipc_tls_stats); 74 75 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 76 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 77 "Kernel TLS offload via OCF stats"); 78 79 static COUNTER_U64_DEFINE_EARLY(ocf_tls10_cbc_encrypts); 80 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls10_cbc_encrypts, 81 CTLFLAG_RD, &ocf_tls10_cbc_encrypts, 82 "Total number of OCF TLS 1.0 CBC encryption operations"); 83 84 static COUNTER_U64_DEFINE_EARLY(ocf_tls11_cbc_encrypts); 85 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_encrypts, 86 CTLFLAG_RD, &ocf_tls11_cbc_encrypts, 87 "Total number of OCF TLS 1.1/1.2 CBC encryption operations"); 88 89 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_decrypts); 90 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_decrypts, 91 CTLFLAG_RD, &ocf_tls12_gcm_decrypts, 92 "Total number of OCF TLS 1.2 GCM decryption operations"); 93 94 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_encrypts); 95 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_encrypts, 96 CTLFLAG_RD, &ocf_tls12_gcm_encrypts, 97 "Total number of OCF TLS 1.2 GCM encryption operations"); 98 99 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_decrypts); 100 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_decrypts, 101 CTLFLAG_RD, &ocf_tls12_chacha20_decrypts, 102 "Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations"); 103 104 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_encrypts); 105 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts, 106 CTLFLAG_RD, &ocf_tls12_chacha20_encrypts, 107 "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); 108 109 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_decrypts); 110 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_decrypts, 111 CTLFLAG_RD, &ocf_tls13_gcm_decrypts, 112 "Total number of OCF TLS 1.3 GCM decryption operations"); 113 114 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts); 115 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts, 116 CTLFLAG_RD, &ocf_tls13_gcm_encrypts, 117 "Total number of OCF TLS 1.3 GCM encryption operations"); 118 119 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_decrypts); 120 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_decrypts, 121 CTLFLAG_RD, &ocf_tls13_chacha20_decrypts, 122 "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations"); 123 124 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts); 125 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts, 126 CTLFLAG_RD, &ocf_tls13_chacha20_encrypts, 127 "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); 128 129 static COUNTER_U64_DEFINE_EARLY(ocf_inplace); 130 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 131 CTLFLAG_RD, &ocf_inplace, 132 "Total number of OCF in-place operations"); 133 134 static COUNTER_U64_DEFINE_EARLY(ocf_separate_output); 135 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 136 CTLFLAG_RD, &ocf_separate_output, 137 "Total number of OCF operations with a separate output buffer"); 138 139 static COUNTER_U64_DEFINE_EARLY(ocf_retries); 140 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 141 &ocf_retries, 142 "Number of OCF encryption operation retries"); 143 144 static int 145 ktls_ocf_callback_sync(struct cryptop *crp __unused) 146 { 147 return (0); 148 } 149 150 static int 151 ktls_ocf_callback_async(struct cryptop *crp) 152 { 153 struct ocf_operation *oo; 154 155 oo = crp->crp_opaque; 156 mtx_lock(&oo->os->lock); 157 oo->done = true; 158 mtx_unlock(&oo->os->lock); 159 wakeup(oo); 160 return (0); 161 } 162 163 static int 164 ktls_ocf_dispatch(struct ktls_ocf_session *os, struct cryptop *crp) 165 { 166 struct ocf_operation oo; 167 int error; 168 bool async; 169 170 oo.os = os; 171 oo.done = false; 172 173 crp->crp_opaque = &oo; 174 for (;;) { 175 async = !CRYPTO_SESS_SYNC(crp->crp_session); 176 crp->crp_callback = async ? ktls_ocf_callback_async : 177 ktls_ocf_callback_sync; 178 179 error = crypto_dispatch(crp); 180 if (error) 181 break; 182 if (async) { 183 mtx_lock(&os->lock); 184 while (!oo.done) 185 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0); 186 mtx_unlock(&os->lock); 187 } 188 189 if (crp->crp_etype != EAGAIN) { 190 error = crp->crp_etype; 191 break; 192 } 193 194 crp->crp_etype = 0; 195 crp->crp_flags &= ~CRYPTO_F_DONE; 196 oo.done = false; 197 counter_u64_add(ocf_retries, 1); 198 } 199 return (error); 200 } 201 202 static int 203 ktls_ocf_dispatch_async_cb(struct cryptop *crp) 204 { 205 struct ktls_ocf_encrypt_state *state; 206 int error; 207 208 state = crp->crp_opaque; 209 if (crp->crp_etype == EAGAIN) { 210 crp->crp_etype = 0; 211 crp->crp_flags &= ~CRYPTO_F_DONE; 212 counter_u64_add(ocf_retries, 1); 213 error = crypto_dispatch(crp); 214 if (error != 0) { 215 crypto_destroyreq(crp); 216 ktls_encrypt_cb(state, error); 217 } 218 return (0); 219 } 220 221 error = crp->crp_etype; 222 crypto_destroyreq(crp); 223 ktls_encrypt_cb(state, error); 224 return (0); 225 } 226 227 static int 228 ktls_ocf_dispatch_async(struct ktls_ocf_encrypt_state *state, 229 struct cryptop *crp) 230 { 231 int error; 232 233 crp->crp_opaque = state; 234 crp->crp_callback = ktls_ocf_dispatch_async_cb; 235 error = crypto_dispatch(crp); 236 if (error != 0) 237 crypto_destroyreq(crp); 238 return (error); 239 } 240 241 static int 242 ktls_ocf_tls_cbc_encrypt(struct ktls_ocf_encrypt_state *state, 243 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 244 int outiovcnt) 245 { 246 const struct tls_record_layer *hdr; 247 struct uio *uio; 248 struct tls_mac_data *ad; 249 struct cryptop *crp; 250 struct ktls_ocf_session *os; 251 struct iovec iov[m->m_epg_npgs + 2]; 252 u_int pgoff; 253 int i, error; 254 uint16_t tls_comp_len; 255 uint8_t pad; 256 257 MPASS(outiovcnt + 1 <= nitems(iov)); 258 259 os = tls->ocf_session; 260 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 261 crp = &state->crp; 262 uio = &state->uio; 263 MPASS(tls->sync_dispatch); 264 265 #ifdef INVARIANTS 266 if (os->implicit_iv) { 267 mtx_lock(&os->lock); 268 KASSERT(!os->in_progress, 269 ("concurrent implicit IV encryptions")); 270 if (os->next_seqno != m->m_epg_seqno) { 271 printf("KTLS CBC: TLS records out of order. " 272 "Expected %ju, got %ju\n", 273 (uintmax_t)os->next_seqno, 274 (uintmax_t)m->m_epg_seqno); 275 mtx_unlock(&os->lock); 276 return (EINVAL); 277 } 278 os->in_progress = true; 279 mtx_unlock(&os->lock); 280 } 281 #endif 282 283 /* Payload length. */ 284 tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen); 285 286 /* Initialize the AAD. */ 287 ad = &state->mac; 288 ad->seq = htobe64(m->m_epg_seqno); 289 ad->type = hdr->tls_type; 290 ad->tls_vmajor = hdr->tls_vmajor; 291 ad->tls_vminor = hdr->tls_vminor; 292 ad->tls_length = htons(tls_comp_len); 293 294 /* First, compute the MAC. */ 295 iov[0].iov_base = ad; 296 iov[0].iov_len = sizeof(*ad); 297 pgoff = m->m_epg_1st_off; 298 for (i = 0; i < m->m_epg_npgs; i++, pgoff = 0) { 299 iov[i + 1].iov_base = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] + 300 pgoff); 301 iov[i + 1].iov_len = m_epg_pagelen(m, i, pgoff); 302 } 303 iov[m->m_epg_npgs + 1].iov_base = m->m_epg_trail; 304 iov[m->m_epg_npgs + 1].iov_len = os->mac_len; 305 uio->uio_iov = iov; 306 uio->uio_iovcnt = m->m_epg_npgs + 2; 307 uio->uio_offset = 0; 308 uio->uio_segflg = UIO_SYSSPACE; 309 uio->uio_td = curthread; 310 uio->uio_resid = sizeof(*ad) + tls_comp_len + os->mac_len; 311 312 crypto_initreq(crp, os->mac_sid); 313 crp->crp_payload_start = 0; 314 crp->crp_payload_length = sizeof(*ad) + tls_comp_len; 315 crp->crp_digest_start = crp->crp_payload_length; 316 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 317 crp->crp_flags = CRYPTO_F_CBIMM; 318 crypto_use_uio(crp, uio); 319 error = ktls_ocf_dispatch(os, crp); 320 321 crypto_destroyreq(crp); 322 if (error) { 323 #ifdef INVARIANTS 324 if (os->implicit_iv) { 325 mtx_lock(&os->lock); 326 os->in_progress = false; 327 mtx_unlock(&os->lock); 328 } 329 #endif 330 return (error); 331 } 332 333 /* Second, add the padding. */ 334 pad = m->m_epg_trllen - os->mac_len - 1; 335 for (i = 0; i < pad + 1; i++) 336 m->m_epg_trail[os->mac_len + i] = pad; 337 338 /* Finally, encrypt the record. */ 339 crypto_initreq(crp, os->sid); 340 crp->crp_payload_start = m->m_epg_hdrlen; 341 crp->crp_payload_length = tls_comp_len + m->m_epg_trllen; 342 KASSERT(crp->crp_payload_length % AES_BLOCK_LEN == 0, 343 ("invalid encryption size")); 344 crypto_use_single_mbuf(crp, m); 345 crp->crp_op = CRYPTO_OP_ENCRYPT; 346 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 347 if (os->implicit_iv) 348 memcpy(crp->crp_iv, os->iv, AES_BLOCK_LEN); 349 else 350 memcpy(crp->crp_iv, hdr + 1, AES_BLOCK_LEN); 351 352 if (outiov != NULL) { 353 uio->uio_iov = outiov; 354 uio->uio_iovcnt = outiovcnt; 355 uio->uio_offset = 0; 356 uio->uio_segflg = UIO_SYSSPACE; 357 uio->uio_td = curthread; 358 uio->uio_resid = crp->crp_payload_length; 359 crypto_use_output_uio(crp, uio); 360 } 361 362 if (os->implicit_iv) 363 counter_u64_add(ocf_tls10_cbc_encrypts, 1); 364 else 365 counter_u64_add(ocf_tls11_cbc_encrypts, 1); 366 if (outiov != NULL) 367 counter_u64_add(ocf_separate_output, 1); 368 else 369 counter_u64_add(ocf_inplace, 1); 370 error = ktls_ocf_dispatch(os, crp); 371 372 crypto_destroyreq(crp); 373 374 if (os->implicit_iv) { 375 KASSERT(os->mac_len + pad + 1 >= AES_BLOCK_LEN, 376 ("trailer too short to read IV")); 377 memcpy(os->iv, m->m_epg_trail + m->m_epg_trllen - AES_BLOCK_LEN, 378 AES_BLOCK_LEN); 379 #ifdef INVARIANTS 380 mtx_lock(&os->lock); 381 os->next_seqno = m->m_epg_seqno + 1; 382 os->in_progress = false; 383 mtx_unlock(&os->lock); 384 #endif 385 } 386 return (error); 387 } 388 389 static int 390 ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state, 391 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 392 int outiovcnt) 393 { 394 const struct tls_record_layer *hdr; 395 struct uio *uio; 396 struct tls_aead_data *ad; 397 struct cryptop *crp; 398 struct ktls_ocf_session *os; 399 int error; 400 uint16_t tls_comp_len; 401 402 os = tls->ocf_session; 403 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 404 crp = &state->crp; 405 uio = &state->uio; 406 407 crypto_initreq(crp, os->sid); 408 409 /* Setup the IV. */ 410 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 411 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 412 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, 413 sizeof(uint64_t)); 414 } else { 415 /* 416 * Chacha20-Poly1305 constructs the IV for TLS 1.2 417 * identically to constructing the IV for AEAD in TLS 418 * 1.3. 419 */ 420 memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len); 421 *(uint64_t *)(crp->crp_iv + 4) ^= htobe64(m->m_epg_seqno); 422 } 423 424 /* Setup the AAD. */ 425 ad = &state->aead; 426 tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen); 427 ad->seq = htobe64(m->m_epg_seqno); 428 ad->type = hdr->tls_type; 429 ad->tls_vmajor = hdr->tls_vmajor; 430 ad->tls_vminor = hdr->tls_vminor; 431 ad->tls_length = htons(tls_comp_len); 432 crp->crp_aad = ad; 433 crp->crp_aad_length = sizeof(*ad); 434 435 /* Set fields for input payload. */ 436 crypto_use_single_mbuf(crp, m); 437 crp->crp_payload_start = m->m_epg_hdrlen; 438 crp->crp_payload_length = tls_comp_len; 439 440 if (outiov != NULL) { 441 crp->crp_digest_start = crp->crp_payload_length; 442 443 uio->uio_iov = outiov; 444 uio->uio_iovcnt = outiovcnt; 445 uio->uio_offset = 0; 446 uio->uio_segflg = UIO_SYSSPACE; 447 uio->uio_td = curthread; 448 uio->uio_resid = crp->crp_payload_length + tls->params.tls_tlen; 449 crypto_use_output_uio(crp, uio); 450 } else 451 crp->crp_digest_start = crp->crp_payload_start + 452 crp->crp_payload_length; 453 454 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 455 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 456 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 457 counter_u64_add(ocf_tls12_gcm_encrypts, 1); 458 else 459 counter_u64_add(ocf_tls12_chacha20_encrypts, 1); 460 if (outiov != NULL) 461 counter_u64_add(ocf_separate_output, 1); 462 else 463 counter_u64_add(ocf_inplace, 1); 464 if (tls->sync_dispatch) { 465 error = ktls_ocf_dispatch(os, crp); 466 crypto_destroyreq(crp); 467 } else 468 error = ktls_ocf_dispatch_async(state, crp); 469 return (error); 470 } 471 472 static int 473 ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, 474 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, 475 int *trailer_len) 476 { 477 struct tls_aead_data ad; 478 struct cryptop crp; 479 struct ktls_ocf_session *os; 480 int error; 481 uint16_t tls_comp_len; 482 483 os = tls->ocf_session; 484 485 crypto_initreq(&crp, os->sid); 486 487 /* Setup the IV. */ 488 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 489 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 490 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, 491 sizeof(uint64_t)); 492 } else { 493 /* 494 * Chacha20-Poly1305 constructs the IV for TLS 1.2 495 * identically to constructing the IV for AEAD in TLS 496 * 1.3. 497 */ 498 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); 499 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); 500 } 501 502 /* Setup the AAD. */ 503 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 504 tls_comp_len = ntohs(hdr->tls_length) - 505 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 506 else 507 tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN; 508 ad.seq = htobe64(seqno); 509 ad.type = hdr->tls_type; 510 ad.tls_vmajor = hdr->tls_vmajor; 511 ad.tls_vminor = hdr->tls_vminor; 512 ad.tls_length = htons(tls_comp_len); 513 crp.crp_aad = &ad; 514 crp.crp_aad_length = sizeof(ad); 515 516 crp.crp_payload_start = tls->params.tls_hlen; 517 crp.crp_payload_length = tls_comp_len; 518 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; 519 520 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; 521 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 522 crypto_use_mbuf(&crp, m); 523 524 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 525 counter_u64_add(ocf_tls12_gcm_decrypts, 1); 526 else 527 counter_u64_add(ocf_tls12_chacha20_decrypts, 1); 528 error = ktls_ocf_dispatch(os, &crp); 529 530 crypto_destroyreq(&crp); 531 *trailer_len = tls->params.tls_tlen; 532 return (error); 533 } 534 535 static int 536 ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, 537 struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, 538 int outiovcnt) 539 { 540 const struct tls_record_layer *hdr; 541 struct uio *uio; 542 struct tls_aead_data_13 *ad; 543 struct cryptop *crp; 544 struct ktls_ocf_session *os; 545 char nonce[12]; 546 int error; 547 548 os = tls->ocf_session; 549 hdr = (const struct tls_record_layer *)m->m_epg_hdr; 550 crp = &state->crp; 551 uio = &state->uio; 552 553 crypto_initreq(crp, os->sid); 554 555 /* Setup the nonce. */ 556 memcpy(nonce, tls->params.iv, tls->params.iv_len); 557 *(uint64_t *)(nonce + 4) ^= htobe64(m->m_epg_seqno); 558 559 /* Setup the AAD. */ 560 ad = &state->aead13; 561 ad->type = hdr->tls_type; 562 ad->tls_vmajor = hdr->tls_vmajor; 563 ad->tls_vminor = hdr->tls_vminor; 564 ad->tls_length = hdr->tls_length; 565 crp->crp_aad = ad; 566 crp->crp_aad_length = sizeof(*ad); 567 568 /* Set fields for input payload. */ 569 crypto_use_single_mbuf(crp, m); 570 crp->crp_payload_start = m->m_epg_hdrlen; 571 crp->crp_payload_length = m->m_len - 572 (m->m_epg_hdrlen + m->m_epg_trllen); 573 574 /* Store the record type as the first byte of the trailer. */ 575 m->m_epg_trail[0] = m->m_epg_record_type; 576 crp->crp_payload_length++; 577 578 if (outiov != NULL) { 579 crp->crp_digest_start = crp->crp_payload_length; 580 581 uio->uio_iov = outiov; 582 uio->uio_iovcnt = outiovcnt; 583 uio->uio_offset = 0; 584 uio->uio_segflg = UIO_SYSSPACE; 585 uio->uio_td = curthread; 586 uio->uio_resid = m->m_len - m->m_epg_hdrlen; 587 crypto_use_output_uio(crp, uio); 588 } else 589 crp->crp_digest_start = crp->crp_payload_start + 590 crp->crp_payload_length; 591 592 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 593 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 594 595 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 596 597 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 598 counter_u64_add(ocf_tls13_gcm_encrypts, 1); 599 else 600 counter_u64_add(ocf_tls13_chacha20_encrypts, 1); 601 if (outiov != NULL) 602 counter_u64_add(ocf_separate_output, 1); 603 else 604 counter_u64_add(ocf_inplace, 1); 605 if (tls->sync_dispatch) { 606 error = ktls_ocf_dispatch(os, crp); 607 crypto_destroyreq(crp); 608 } else 609 error = ktls_ocf_dispatch_async(state, crp); 610 return (error); 611 } 612 613 static int 614 ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls, 615 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, 616 int *trailer_len) 617 { 618 struct tls_aead_data_13 ad; 619 struct cryptop crp; 620 struct ktls_ocf_session *os; 621 int error; 622 u_int tag_len; 623 624 os = tls->ocf_session; 625 626 tag_len = tls->params.tls_tlen - 1; 627 628 /* Payload must contain at least one byte for the record type. */ 629 if (ntohs(hdr->tls_length) < tag_len + 1) 630 return (EBADMSG); 631 632 crypto_initreq(&crp, os->sid); 633 634 /* Setup the nonce. */ 635 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len); 636 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno); 637 638 /* Setup the AAD. */ 639 ad.type = hdr->tls_type; 640 ad.tls_vmajor = hdr->tls_vmajor; 641 ad.tls_vminor = hdr->tls_vminor; 642 ad.tls_length = hdr->tls_length; 643 crp.crp_aad = &ad; 644 crp.crp_aad_length = sizeof(ad); 645 646 crp.crp_payload_start = tls->params.tls_hlen; 647 crp.crp_payload_length = ntohs(hdr->tls_length) - tag_len; 648 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; 649 650 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; 651 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 652 crypto_use_mbuf(&crp, m); 653 654 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 655 counter_u64_add(ocf_tls13_gcm_decrypts, 1); 656 else 657 counter_u64_add(ocf_tls13_chacha20_decrypts, 1); 658 error = ktls_ocf_dispatch(os, &crp); 659 660 crypto_destroyreq(&crp); 661 *trailer_len = tag_len; 662 return (error); 663 } 664 665 void 666 ktls_ocf_free(struct ktls_session *tls) 667 { 668 struct ktls_ocf_session *os; 669 670 os = tls->ocf_session; 671 crypto_freesession(os->sid); 672 mtx_destroy(&os->lock); 673 zfree(os, M_KTLS_OCF); 674 } 675 676 int 677 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) 678 { 679 struct crypto_session_params csp, mac_csp; 680 struct ktls_ocf_session *os; 681 int error, mac_len; 682 683 memset(&csp, 0, sizeof(csp)); 684 memset(&mac_csp, 0, sizeof(mac_csp)); 685 mac_csp.csp_mode = CSP_MODE_NONE; 686 mac_len = 0; 687 688 switch (tls->params.cipher_algorithm) { 689 case CRYPTO_AES_NIST_GCM_16: 690 switch (tls->params.cipher_key_len) { 691 case 128 / 8: 692 case 256 / 8: 693 break; 694 default: 695 return (EINVAL); 696 } 697 698 /* Only TLS 1.2 and 1.3 are supported. */ 699 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 700 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 701 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 702 return (EPROTONOSUPPORT); 703 704 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 705 csp.csp_mode = CSP_MODE_AEAD; 706 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 707 csp.csp_cipher_key = tls->params.cipher_key; 708 csp.csp_cipher_klen = tls->params.cipher_key_len; 709 csp.csp_ivlen = AES_GCM_IV_LEN; 710 break; 711 case CRYPTO_AES_CBC: 712 switch (tls->params.cipher_key_len) { 713 case 128 / 8: 714 case 256 / 8: 715 break; 716 default: 717 return (EINVAL); 718 } 719 720 switch (tls->params.auth_algorithm) { 721 case CRYPTO_SHA1_HMAC: 722 mac_len = SHA1_HASH_LEN; 723 break; 724 case CRYPTO_SHA2_256_HMAC: 725 mac_len = SHA2_256_HASH_LEN; 726 break; 727 case CRYPTO_SHA2_384_HMAC: 728 mac_len = SHA2_384_HASH_LEN; 729 break; 730 default: 731 return (EINVAL); 732 } 733 734 /* Only TLS 1.0-1.2 are supported. */ 735 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 736 tls->params.tls_vminor < TLS_MINOR_VER_ZERO || 737 tls->params.tls_vminor > TLS_MINOR_VER_TWO) 738 return (EPROTONOSUPPORT); 739 740 /* AES-CBC is not supported for receive. */ 741 if (direction == KTLS_RX) 742 return (EPROTONOSUPPORT); 743 744 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 745 csp.csp_mode = CSP_MODE_CIPHER; 746 csp.csp_cipher_alg = CRYPTO_AES_CBC; 747 csp.csp_cipher_key = tls->params.cipher_key; 748 csp.csp_cipher_klen = tls->params.cipher_key_len; 749 csp.csp_ivlen = AES_BLOCK_LEN; 750 751 mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 752 mac_csp.csp_mode = CSP_MODE_DIGEST; 753 mac_csp.csp_auth_alg = tls->params.auth_algorithm; 754 mac_csp.csp_auth_key = tls->params.auth_key; 755 mac_csp.csp_auth_klen = tls->params.auth_key_len; 756 break; 757 case CRYPTO_CHACHA20_POLY1305: 758 switch (tls->params.cipher_key_len) { 759 case 256 / 8: 760 break; 761 default: 762 return (EINVAL); 763 } 764 765 /* Only TLS 1.2 and 1.3 are supported. */ 766 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 767 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 768 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 769 return (EPROTONOSUPPORT); 770 771 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 772 csp.csp_mode = CSP_MODE_AEAD; 773 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 774 csp.csp_cipher_key = tls->params.cipher_key; 775 csp.csp_cipher_klen = tls->params.cipher_key_len; 776 csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN; 777 break; 778 default: 779 return (EPROTONOSUPPORT); 780 } 781 782 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 783 if (os == NULL) 784 return (ENOMEM); 785 786 error = crypto_newsession(&os->sid, &csp, 787 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 788 if (error) { 789 free(os, M_KTLS_OCF); 790 return (error); 791 } 792 793 if (mac_csp.csp_mode != CSP_MODE_NONE) { 794 error = crypto_newsession(&os->mac_sid, &mac_csp, 795 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 796 if (error) { 797 crypto_freesession(os->sid); 798 free(os, M_KTLS_OCF); 799 return (error); 800 } 801 os->mac_len = mac_len; 802 } 803 804 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 805 tls->ocf_session = os; 806 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 || 807 tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { 808 if (direction == KTLS_TX) { 809 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 810 tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; 811 else 812 tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; 813 } else { 814 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 815 tls->sw_decrypt = ktls_ocf_tls13_aead_decrypt; 816 else 817 tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt; 818 } 819 } else { 820 tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt; 821 if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) { 822 os->implicit_iv = true; 823 memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN); 824 #ifdef INVARIANTS 825 os->next_seqno = tls->next_seqno; 826 #endif 827 } 828 } 829 830 /* 831 * AES-CBC is always synchronous currently. Asynchronous 832 * operation would require multiple callbacks and an additional 833 * iovec array in ktls_ocf_encrypt_state. 834 */ 835 tls->sync_dispatch = CRYPTO_SESS_SYNC(os->sid) || 836 tls->params.cipher_algorithm == CRYPTO_AES_CBC; 837 return (0); 838 } 839