1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 struct mtx lock; 49 }; 50 51 struct ocf_operation { 52 struct ocf_session *os; 53 bool done; 54 struct iovec iov[0]; 55 }; 56 57 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 58 59 SYSCTL_DECL(_kern_ipc_tls); 60 SYSCTL_DECL(_kern_ipc_tls_stats); 61 62 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 63 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 64 "Kernel TLS offload via OCF stats"); 65 66 static counter_u64_t ocf_tls12_gcm_crypts; 67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 68 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 69 "Total number of OCF TLS 1.2 GCM encryption operations"); 70 71 static counter_u64_t ocf_tls13_gcm_crypts; 72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 73 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 74 "Total number of OCF TLS 1.3 GCM encryption operations"); 75 76 static counter_u64_t ocf_inplace; 77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 78 CTLFLAG_RD, &ocf_inplace, 79 "Total number of OCF in-place operations"); 80 81 static counter_u64_t ocf_separate_output; 82 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 83 CTLFLAG_RD, &ocf_separate_output, 84 "Total number of OCF operations with a separate output buffer"); 85 86 static counter_u64_t ocf_retries; 87 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 88 &ocf_retries, 89 "Number of OCF encryption operation retries"); 90 91 static int 92 ktls_ocf_callback(struct cryptop *crp) 93 { 94 struct ocf_operation *oo; 95 96 oo = crp->crp_opaque; 97 mtx_lock(&oo->os->lock); 98 oo->done = true; 99 mtx_unlock(&oo->os->lock); 100 wakeup(oo); 101 return (0); 102 } 103 104 static int 105 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 106 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 107 struct iovec *outiov, int iovcnt, uint64_t seqno, 108 uint8_t record_type __unused) 109 { 110 struct uio uio, out_uio, *tag_uio; 111 struct tls_aead_data ad; 112 struct cryptop *crp; 113 struct ocf_session *os; 114 struct ocf_operation *oo; 115 struct iovec *iov, *out_iov; 116 int i, error; 117 uint16_t tls_comp_len; 118 bool inplace; 119 120 os = tls->cipher; 121 122 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov) * 2, M_KTLS_OCF, 123 M_WAITOK | M_ZERO); 124 oo->os = os; 125 iov = oo->iov; 126 out_iov = iov + iovcnt + 2; 127 128 uio.uio_iov = iov; 129 uio.uio_offset = 0; 130 uio.uio_segflg = UIO_SYSSPACE; 131 uio.uio_td = curthread; 132 133 out_uio.uio_iov = out_iov; 134 out_uio.uio_offset = 0; 135 out_uio.uio_segflg = UIO_SYSSPACE; 136 out_uio.uio_td = curthread; 137 138 crp = crypto_getreq(os->sid, M_WAITOK); 139 140 /* Setup the IV. */ 141 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 142 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 143 144 /* Setup the AAD. */ 145 tls_comp_len = ntohs(hdr->tls_length) - 146 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 147 ad.seq = htobe64(seqno); 148 ad.type = hdr->tls_type; 149 ad.tls_vmajor = hdr->tls_vmajor; 150 ad.tls_vminor = hdr->tls_vminor; 151 ad.tls_length = htons(tls_comp_len); 152 iov[0].iov_base = &ad; 153 iov[0].iov_len = sizeof(ad); 154 crp->crp_aad_start = 0; 155 crp->crp_aad_length = sizeof(ad); 156 157 /* Copy iov's. */ 158 memcpy(iov + 1, iniov, iovcnt * sizeof(*iov)); 159 uio.uio_iovcnt = iovcnt + 1; 160 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); 161 out_uio.uio_iovcnt = iovcnt; 162 163 /* Compute payload length and determine if encryption is in place. */ 164 inplace = true; 165 crp->crp_payload_start = sizeof(ad); 166 for (i = 0; i < iovcnt; i++) { 167 if (iniov[i].iov_base != outiov[i].iov_base) 168 inplace = false; 169 crp->crp_payload_length += iniov[i].iov_len; 170 } 171 uio.uio_resid = sizeof(ad) + crp->crp_payload_length; 172 out_uio.uio_resid = crp->crp_payload_length; 173 174 if (inplace) 175 tag_uio = &uio; 176 else 177 tag_uio = &out_uio; 178 179 tag_uio->uio_iov[tag_uio->uio_iovcnt].iov_base = trailer; 180 tag_uio->uio_iov[tag_uio->uio_iovcnt].iov_len = AES_GMAC_HASH_LEN; 181 tag_uio->uio_iovcnt++; 182 crp->crp_digest_start = tag_uio->uio_resid; 183 tag_uio->uio_resid += AES_GMAC_HASH_LEN; 184 185 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 186 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 187 crypto_use_uio(crp, &uio); 188 if (!inplace) 189 crypto_use_output_uio(crp, &out_uio); 190 crp->crp_opaque = oo; 191 crp->crp_callback = ktls_ocf_callback; 192 193 counter_u64_add(ocf_tls12_gcm_crypts, 1); 194 if (inplace) 195 counter_u64_add(ocf_inplace, 1); 196 else 197 counter_u64_add(ocf_separate_output, 1); 198 for (;;) { 199 error = crypto_dispatch(crp); 200 if (error) 201 break; 202 203 mtx_lock(&os->lock); 204 while (!oo->done) 205 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 206 mtx_unlock(&os->lock); 207 208 if (crp->crp_etype != EAGAIN) { 209 error = crp->crp_etype; 210 break; 211 } 212 213 crp->crp_etype = 0; 214 crp->crp_flags &= ~CRYPTO_F_DONE; 215 oo->done = false; 216 counter_u64_add(ocf_retries, 1); 217 } 218 219 crypto_freereq(crp); 220 free(oo, M_KTLS_OCF); 221 return (error); 222 } 223 224 static int 225 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 226 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 227 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 228 { 229 struct uio uio, out_uio; 230 struct tls_aead_data_13 ad; 231 char nonce[12]; 232 struct cryptop *crp; 233 struct ocf_session *os; 234 struct ocf_operation *oo; 235 struct iovec *iov, *out_iov; 236 int i, error; 237 bool inplace; 238 239 os = tls->cipher; 240 241 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov) * 2, M_KTLS_OCF, 242 M_WAITOK | M_ZERO); 243 oo->os = os; 244 iov = oo->iov; 245 246 out_iov = iov + iovcnt + 2; 247 248 uio.uio_iov = iov; 249 uio.uio_offset = 0; 250 uio.uio_segflg = UIO_SYSSPACE; 251 uio.uio_td = curthread; 252 253 out_uio.uio_iov = out_iov; 254 out_uio.uio_offset = 0; 255 out_uio.uio_segflg = UIO_SYSSPACE; 256 out_uio.uio_td = curthread; 257 258 crp = crypto_getreq(os->sid, M_WAITOK); 259 260 /* Setup the nonce. */ 261 memcpy(nonce, tls->params.iv, tls->params.iv_len); 262 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 263 264 /* Setup the AAD. */ 265 ad.type = hdr->tls_type; 266 ad.tls_vmajor = hdr->tls_vmajor; 267 ad.tls_vminor = hdr->tls_vminor; 268 ad.tls_length = hdr->tls_length; 269 iov[0].iov_base = &ad; 270 iov[0].iov_len = sizeof(ad); 271 crp->crp_aad_start = 0; 272 crp->crp_aad_length = sizeof(ad); 273 274 /* Copy iov's. */ 275 memcpy(iov + 1, iniov, iovcnt * sizeof(*iov)); 276 uio.uio_iovcnt = iovcnt + 1; 277 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); 278 out_uio.uio_iovcnt = iovcnt; 279 280 /* Compute payload length and determine if encryption is in place. */ 281 inplace = true; 282 crp->crp_payload_start = sizeof(ad); 283 for (i = 0; i < iovcnt; i++) { 284 if (iniov[i].iov_base != outiov[i].iov_base) 285 inplace = false; 286 crp->crp_payload_length += iniov[i].iov_len; 287 } 288 uio.uio_resid = sizeof(ad) + crp->crp_payload_length; 289 out_uio.uio_resid = crp->crp_payload_length; 290 291 /* 292 * Always include the full trailer as input to get the 293 * record_type even if only the first byte is used. 294 */ 295 trailer[0] = record_type; 296 crp->crp_payload_length++; 297 iov[iovcnt + 1].iov_base = trailer; 298 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1; 299 uio.uio_iovcnt++; 300 uio.uio_resid += AES_GMAC_HASH_LEN + 1; 301 if (inplace) { 302 crp->crp_digest_start = uio.uio_resid - AES_GMAC_HASH_LEN; 303 } else { 304 out_iov[iovcnt] = iov[iovcnt + 1]; 305 out_uio.uio_iovcnt++; 306 out_uio.uio_resid += AES_GMAC_HASH_LEN + 1; 307 crp->crp_digest_start = out_uio.uio_resid - AES_GMAC_HASH_LEN; 308 } 309 310 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 311 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 312 crypto_use_uio(crp, &uio); 313 if (!inplace) 314 crypto_use_output_uio(crp, &out_uio); 315 crp->crp_opaque = oo; 316 crp->crp_callback = ktls_ocf_callback; 317 318 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 319 320 counter_u64_add(ocf_tls13_gcm_crypts, 1); 321 if (inplace) 322 counter_u64_add(ocf_inplace, 1); 323 else 324 counter_u64_add(ocf_separate_output, 1); 325 for (;;) { 326 error = crypto_dispatch(crp); 327 if (error) 328 break; 329 330 mtx_lock(&os->lock); 331 while (!oo->done) 332 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 333 mtx_unlock(&os->lock); 334 335 if (crp->crp_etype != EAGAIN) { 336 error = crp->crp_etype; 337 break; 338 } 339 340 crp->crp_etype = 0; 341 crp->crp_flags &= ~CRYPTO_F_DONE; 342 oo->done = false; 343 counter_u64_add(ocf_retries, 1); 344 } 345 346 crypto_freereq(crp); 347 free(oo, M_KTLS_OCF); 348 return (error); 349 } 350 351 static void 352 ktls_ocf_free(struct ktls_session *tls) 353 { 354 struct ocf_session *os; 355 356 os = tls->cipher; 357 crypto_freesession(os->sid); 358 mtx_destroy(&os->lock); 359 explicit_bzero(os, sizeof(*os)); 360 free(os, M_KTLS_OCF); 361 } 362 363 static int 364 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 365 { 366 struct crypto_session_params csp; 367 struct ocf_session *os; 368 int error; 369 370 memset(&csp, 0, sizeof(csp)); 371 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 372 373 switch (tls->params.cipher_algorithm) { 374 case CRYPTO_AES_NIST_GCM_16: 375 switch (tls->params.cipher_key_len) { 376 case 128 / 8: 377 case 256 / 8: 378 break; 379 default: 380 return (EINVAL); 381 } 382 csp.csp_mode = CSP_MODE_AEAD; 383 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 384 csp.csp_cipher_key = tls->params.cipher_key; 385 csp.csp_cipher_klen = tls->params.cipher_key_len; 386 csp.csp_ivlen = AES_GCM_IV_LEN; 387 break; 388 default: 389 return (EPROTONOSUPPORT); 390 } 391 392 /* Only TLS 1.2 and 1.3 are supported. */ 393 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 394 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 395 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 396 return (EPROTONOSUPPORT); 397 398 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 399 if (os == NULL) 400 return (ENOMEM); 401 402 error = crypto_newsession(&os->sid, &csp, 403 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 404 if (error) { 405 free(os, M_KTLS_OCF); 406 return (error); 407 } 408 409 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 410 tls->cipher = os; 411 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 412 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 413 else 414 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 415 tls->free = ktls_ocf_free; 416 return (0); 417 } 418 419 struct ktls_crypto_backend ocf_backend = { 420 .name = "OCF", 421 .prio = 5, 422 .api_version = KTLS_API_VERSION, 423 .try = ktls_ocf_try, 424 }; 425 426 static int 427 ktls_ocf_modevent(module_t mod, int what, void *arg) 428 { 429 int error; 430 431 switch (what) { 432 case MOD_LOAD: 433 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 434 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 435 ocf_inplace = counter_u64_alloc(M_WAITOK); 436 ocf_separate_output = counter_u64_alloc(M_WAITOK); 437 ocf_retries = counter_u64_alloc(M_WAITOK); 438 return (ktls_crypto_backend_register(&ocf_backend)); 439 case MOD_UNLOAD: 440 error = ktls_crypto_backend_deregister(&ocf_backend); 441 if (error) 442 return (error); 443 counter_u64_free(ocf_tls12_gcm_crypts); 444 counter_u64_free(ocf_tls13_gcm_crypts); 445 counter_u64_free(ocf_inplace); 446 counter_u64_free(ocf_separate_output); 447 counter_u64_free(ocf_retries); 448 return (0); 449 default: 450 return (EOPNOTSUPP); 451 } 452 } 453 454 static moduledata_t ktls_ocf_moduledata = { 455 "ktls_ocf", 456 ktls_ocf_modevent, 457 NULL 458 }; 459 460 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 461