1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 struct mtx lock; 49 }; 50 51 struct ocf_operation { 52 struct ocf_session *os; 53 bool done; 54 }; 55 56 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 57 58 SYSCTL_DECL(_kern_ipc_tls); 59 SYSCTL_DECL(_kern_ipc_tls_stats); 60 61 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 62 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 63 "Kernel TLS offload via OCF stats"); 64 65 static counter_u64_t ocf_tls12_gcm_crypts; 66 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 67 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 68 "Total number of OCF TLS 1.2 GCM encryption operations"); 69 70 static counter_u64_t ocf_tls13_gcm_crypts; 71 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 72 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 73 "Total number of OCF TLS 1.3 GCM encryption operations"); 74 75 static counter_u64_t ocf_inplace; 76 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 77 CTLFLAG_RD, &ocf_inplace, 78 "Total number of OCF in-place operations"); 79 80 static counter_u64_t ocf_separate_output; 81 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 82 CTLFLAG_RD, &ocf_separate_output, 83 "Total number of OCF operations with a separate output buffer"); 84 85 static counter_u64_t ocf_retries; 86 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 87 &ocf_retries, 88 "Number of OCF encryption operation retries"); 89 90 static int 91 ktls_ocf_callback(struct cryptop *crp) 92 { 93 struct ocf_operation *oo; 94 95 oo = crp->crp_opaque; 96 mtx_lock(&oo->os->lock); 97 oo->done = true; 98 mtx_unlock(&oo->os->lock); 99 wakeup(oo); 100 return (0); 101 } 102 103 static int 104 ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp) 105 { 106 struct ocf_operation oo; 107 int error; 108 109 oo.os = os; 110 oo.done = false; 111 112 crp->crp_opaque = &oo; 113 crp->crp_callback = ktls_ocf_callback; 114 for (;;) { 115 error = crypto_dispatch(crp); 116 if (error) 117 break; 118 119 mtx_lock(&os->lock); 120 while (!oo.done) 121 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0); 122 mtx_unlock(&os->lock); 123 124 if (crp->crp_etype != EAGAIN) { 125 error = crp->crp_etype; 126 break; 127 } 128 129 crp->crp_etype = 0; 130 crp->crp_flags &= ~CRYPTO_F_DONE; 131 oo.done = false; 132 counter_u64_add(ocf_retries, 1); 133 } 134 return (error); 135 } 136 137 static int 138 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 139 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 140 struct iovec *outiov, int iovcnt, uint64_t seqno, 141 uint8_t record_type __unused) 142 { 143 struct uio uio, out_uio, *tag_uio; 144 struct tls_aead_data ad; 145 struct cryptop crp; 146 struct ocf_session *os; 147 struct iovec iov[iovcnt + 1]; 148 int i, error; 149 uint16_t tls_comp_len; 150 bool inplace; 151 152 os = tls->cipher; 153 154 uio.uio_iov = iniov; 155 uio.uio_iovcnt = iovcnt; 156 uio.uio_offset = 0; 157 uio.uio_segflg = UIO_SYSSPACE; 158 uio.uio_td = curthread; 159 160 out_uio.uio_iov = outiov; 161 out_uio.uio_iovcnt = iovcnt; 162 out_uio.uio_offset = 0; 163 out_uio.uio_segflg = UIO_SYSSPACE; 164 out_uio.uio_td = curthread; 165 166 crypto_initreq(&crp, os->sid); 167 168 /* Setup the IV. */ 169 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 170 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 171 172 /* Setup the AAD. */ 173 tls_comp_len = ntohs(hdr->tls_length) - 174 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 175 ad.seq = htobe64(seqno); 176 ad.type = hdr->tls_type; 177 ad.tls_vmajor = hdr->tls_vmajor; 178 ad.tls_vminor = hdr->tls_vminor; 179 ad.tls_length = htons(tls_comp_len); 180 crp.crp_aad = &ad; 181 crp.crp_aad_length = sizeof(ad); 182 183 /* Compute payload length and determine if encryption is in place. */ 184 inplace = true; 185 crp.crp_payload_start = 0; 186 for (i = 0; i < iovcnt; i++) { 187 if (iniov[i].iov_base != outiov[i].iov_base) 188 inplace = false; 189 crp.crp_payload_length += iniov[i].iov_len; 190 } 191 uio.uio_resid = crp.crp_payload_length; 192 out_uio.uio_resid = crp.crp_payload_length; 193 194 if (inplace) 195 tag_uio = &uio; 196 else 197 tag_uio = &out_uio; 198 199 /* Duplicate iovec and append vector for tag. */ 200 memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec)); 201 iov[iovcnt].iov_base = trailer; 202 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN; 203 tag_uio->uio_iov = iov; 204 tag_uio->uio_iovcnt++; 205 crp.crp_digest_start = tag_uio->uio_resid; 206 tag_uio->uio_resid += AES_GMAC_HASH_LEN; 207 208 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 209 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 210 crypto_use_uio(&crp, &uio); 211 if (!inplace) 212 crypto_use_output_uio(&crp, &out_uio); 213 214 counter_u64_add(ocf_tls12_gcm_crypts, 1); 215 if (inplace) 216 counter_u64_add(ocf_inplace, 1); 217 else 218 counter_u64_add(ocf_separate_output, 1); 219 error = ktls_ocf_dispatch(os, &crp); 220 221 crypto_destroyreq(&crp); 222 return (error); 223 } 224 225 static int 226 ktls_ocf_tls12_gcm_decrypt(struct ktls_session *tls, 227 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, 228 int *trailer_len) 229 { 230 struct tls_aead_data ad; 231 struct cryptop crp; 232 struct ocf_session *os; 233 struct ocf_operation oo; 234 int error; 235 uint16_t tls_comp_len; 236 237 os = tls->cipher; 238 239 oo.os = os; 240 oo.done = false; 241 242 crypto_initreq(&crp, os->sid); 243 244 /* Setup the IV. */ 245 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 246 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 247 248 /* Setup the AAD. */ 249 tls_comp_len = ntohs(hdr->tls_length) - 250 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 251 ad.seq = htobe64(seqno); 252 ad.type = hdr->tls_type; 253 ad.tls_vmajor = hdr->tls_vmajor; 254 ad.tls_vminor = hdr->tls_vminor; 255 ad.tls_length = htons(tls_comp_len); 256 crp.crp_aad = &ad; 257 crp.crp_aad_length = sizeof(ad); 258 259 crp.crp_payload_start = tls->params.tls_hlen; 260 crp.crp_payload_length = tls_comp_len; 261 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; 262 263 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; 264 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 265 crypto_use_mbuf(&crp, m); 266 267 counter_u64_add(ocf_tls12_gcm_crypts, 1); 268 error = ktls_ocf_dispatch(os, &crp); 269 270 crypto_destroyreq(&crp); 271 *trailer_len = AES_GMAC_HASH_LEN; 272 return (error); 273 } 274 275 static int 276 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 277 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 278 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 279 { 280 struct uio uio, out_uio; 281 struct tls_aead_data_13 ad; 282 char nonce[12]; 283 struct cryptop crp; 284 struct ocf_session *os; 285 struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1]; 286 int i, error; 287 bool inplace; 288 289 os = tls->cipher; 290 291 crypto_initreq(&crp, os->sid); 292 293 /* Setup the nonce. */ 294 memcpy(nonce, tls->params.iv, tls->params.iv_len); 295 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 296 297 /* Setup the AAD. */ 298 ad.type = hdr->tls_type; 299 ad.tls_vmajor = hdr->tls_vmajor; 300 ad.tls_vminor = hdr->tls_vminor; 301 ad.tls_length = hdr->tls_length; 302 crp.crp_aad = &ad; 303 crp.crp_aad_length = sizeof(ad); 304 305 /* Compute payload length and determine if encryption is in place. */ 306 inplace = true; 307 crp.crp_payload_start = 0; 308 for (i = 0; i < iovcnt; i++) { 309 if (iniov[i].iov_base != outiov[i].iov_base) 310 inplace = false; 311 crp.crp_payload_length += iniov[i].iov_len; 312 } 313 314 /* Store the record type as the first byte of the trailer. */ 315 trailer[0] = record_type; 316 crp.crp_payload_length++; 317 crp.crp_digest_start = crp.crp_payload_length; 318 319 /* 320 * Duplicate the input iov to append the trailer. Always 321 * include the full trailer as input to get the record_type 322 * even if only the first byte is used. 323 */ 324 memcpy(iov, iniov, iovcnt * sizeof(*iov)); 325 iov[iovcnt].iov_base = trailer; 326 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1; 327 uio.uio_iov = iov; 328 uio.uio_iovcnt = iovcnt + 1; 329 uio.uio_offset = 0; 330 uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN; 331 uio.uio_segflg = UIO_SYSSPACE; 332 uio.uio_td = curthread; 333 crypto_use_uio(&crp, &uio); 334 335 if (!inplace) { 336 /* Duplicate the output iov to append the trailer. */ 337 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); 338 out_iov[iovcnt] = iov[iovcnt]; 339 340 out_uio.uio_iov = out_iov; 341 out_uio.uio_iovcnt = iovcnt + 1; 342 out_uio.uio_offset = 0; 343 out_uio.uio_resid = crp.crp_payload_length + 344 AES_GMAC_HASH_LEN; 345 out_uio.uio_segflg = UIO_SYSSPACE; 346 out_uio.uio_td = curthread; 347 crypto_use_output_uio(&crp, &out_uio); 348 } 349 350 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 351 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 352 353 memcpy(crp.crp_iv, nonce, sizeof(nonce)); 354 355 counter_u64_add(ocf_tls13_gcm_crypts, 1); 356 if (inplace) 357 counter_u64_add(ocf_inplace, 1); 358 else 359 counter_u64_add(ocf_separate_output, 1); 360 error = ktls_ocf_dispatch(os, &crp); 361 362 crypto_destroyreq(&crp); 363 return (error); 364 } 365 366 static void 367 ktls_ocf_free(struct ktls_session *tls) 368 { 369 struct ocf_session *os; 370 371 os = tls->cipher; 372 crypto_freesession(os->sid); 373 mtx_destroy(&os->lock); 374 zfree(os, M_KTLS_OCF); 375 } 376 377 static int 378 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) 379 { 380 struct crypto_session_params csp; 381 struct ocf_session *os; 382 int error; 383 384 memset(&csp, 0, sizeof(csp)); 385 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 386 387 switch (tls->params.cipher_algorithm) { 388 case CRYPTO_AES_NIST_GCM_16: 389 switch (tls->params.cipher_key_len) { 390 case 128 / 8: 391 case 256 / 8: 392 break; 393 default: 394 return (EINVAL); 395 } 396 csp.csp_mode = CSP_MODE_AEAD; 397 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 398 csp.csp_cipher_key = tls->params.cipher_key; 399 csp.csp_cipher_klen = tls->params.cipher_key_len; 400 csp.csp_ivlen = AES_GCM_IV_LEN; 401 break; 402 default: 403 return (EPROTONOSUPPORT); 404 } 405 406 /* Only TLS 1.2 and 1.3 are supported. */ 407 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 408 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 409 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 410 return (EPROTONOSUPPORT); 411 412 /* TLS 1.3 is not yet supported for receive. */ 413 if (direction == KTLS_RX && 414 tls->params.tls_vminor == TLS_MINOR_VER_THREE) 415 return (EPROTONOSUPPORT); 416 417 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 418 if (os == NULL) 419 return (ENOMEM); 420 421 error = crypto_newsession(&os->sid, &csp, 422 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 423 if (error) { 424 free(os, M_KTLS_OCF); 425 return (error); 426 } 427 428 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 429 tls->cipher = os; 430 if (direction == KTLS_TX) { 431 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 432 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 433 else 434 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 435 } else { 436 tls->sw_decrypt = ktls_ocf_tls12_gcm_decrypt; 437 } 438 tls->free = ktls_ocf_free; 439 return (0); 440 } 441 442 struct ktls_crypto_backend ocf_backend = { 443 .name = "OCF", 444 .prio = 5, 445 .api_version = KTLS_API_VERSION, 446 .try = ktls_ocf_try, 447 }; 448 449 static int 450 ktls_ocf_modevent(module_t mod, int what, void *arg) 451 { 452 int error; 453 454 switch (what) { 455 case MOD_LOAD: 456 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 457 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 458 ocf_inplace = counter_u64_alloc(M_WAITOK); 459 ocf_separate_output = counter_u64_alloc(M_WAITOK); 460 ocf_retries = counter_u64_alloc(M_WAITOK); 461 return (ktls_crypto_backend_register(&ocf_backend)); 462 case MOD_UNLOAD: 463 error = ktls_crypto_backend_deregister(&ocf_backend); 464 if (error) 465 return (error); 466 counter_u64_free(ocf_tls12_gcm_crypts); 467 counter_u64_free(ocf_tls13_gcm_crypts); 468 counter_u64_free(ocf_inplace); 469 counter_u64_free(ocf_separate_output); 470 counter_u64_free(ocf_retries); 471 return (0); 472 default: 473 return (EOPNOTSUPP); 474 } 475 } 476 477 static moduledata_t ktls_ocf_moduledata = { 478 "ktls_ocf", 479 ktls_ocf_modevent, 480 NULL 481 }; 482 483 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 484