1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 struct mtx lock; 49 }; 50 51 struct ocf_operation { 52 struct ocf_session *os; 53 bool done; 54 struct iovec iov[0]; 55 }; 56 57 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 58 59 SYSCTL_DECL(_kern_ipc_tls); 60 SYSCTL_DECL(_kern_ipc_tls_stats); 61 62 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 63 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 64 "Kernel TLS offload via OCF stats"); 65 66 static counter_u64_t ocf_tls12_gcm_crypts; 67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 68 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 69 "Total number of OCF TLS 1.2 GCM encryption operations"); 70 71 static counter_u64_t ocf_tls13_gcm_crypts; 72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 73 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 74 "Total number of OCF TLS 1.3 GCM encryption operations"); 75 76 static counter_u64_t ocf_inplace; 77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace, 78 CTLFLAG_RD, &ocf_inplace, 79 "Total number of OCF in-place operations"); 80 81 static counter_u64_t ocf_separate_output; 82 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output, 83 CTLFLAG_RD, &ocf_separate_output, 84 "Total number of OCF operations with a separate output buffer"); 85 86 static counter_u64_t ocf_retries; 87 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 88 &ocf_retries, 89 "Number of OCF encryption operation retries"); 90 91 static int 92 ktls_ocf_callback(struct cryptop *crp) 93 { 94 struct ocf_operation *oo; 95 96 oo = crp->crp_opaque; 97 mtx_lock(&oo->os->lock); 98 oo->done = true; 99 mtx_unlock(&oo->os->lock); 100 wakeup(oo); 101 return (0); 102 } 103 104 static int 105 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 106 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 107 struct iovec *outiov, int iovcnt, uint64_t seqno, 108 uint8_t record_type __unused) 109 { 110 struct uio uio, out_uio, *tag_uio; 111 struct tls_aead_data ad; 112 struct cryptop *crp; 113 struct ocf_session *os; 114 struct ocf_operation *oo; 115 int i, error; 116 uint16_t tls_comp_len; 117 bool inplace; 118 119 os = tls->cipher; 120 121 oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(struct iovec), 122 M_KTLS_OCF, M_WAITOK | M_ZERO); 123 oo->os = os; 124 125 uio.uio_iov = iniov; 126 uio.uio_iovcnt = iovcnt; 127 uio.uio_offset = 0; 128 uio.uio_segflg = UIO_SYSSPACE; 129 uio.uio_td = curthread; 130 131 out_uio.uio_iov = outiov; 132 out_uio.uio_iovcnt = iovcnt; 133 out_uio.uio_offset = 0; 134 out_uio.uio_segflg = UIO_SYSSPACE; 135 out_uio.uio_td = curthread; 136 137 crp = crypto_getreq(os->sid, M_WAITOK); 138 139 /* Setup the IV. */ 140 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 141 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 142 143 /* Setup the AAD. */ 144 tls_comp_len = ntohs(hdr->tls_length) - 145 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 146 ad.seq = htobe64(seqno); 147 ad.type = hdr->tls_type; 148 ad.tls_vmajor = hdr->tls_vmajor; 149 ad.tls_vminor = hdr->tls_vminor; 150 ad.tls_length = htons(tls_comp_len); 151 crp->crp_aad = &ad; 152 crp->crp_aad_length = sizeof(ad); 153 154 /* Compute payload length and determine if encryption is in place. */ 155 inplace = true; 156 crp->crp_payload_start = 0; 157 for (i = 0; i < iovcnt; i++) { 158 if (iniov[i].iov_base != outiov[i].iov_base) 159 inplace = false; 160 crp->crp_payload_length += iniov[i].iov_len; 161 } 162 uio.uio_resid = crp->crp_payload_length; 163 out_uio.uio_resid = crp->crp_payload_length; 164 165 if (inplace) 166 tag_uio = &uio; 167 else 168 tag_uio = &out_uio; 169 170 /* Duplicate iovec and append vector for tag. */ 171 memcpy(oo->iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec)); 172 tag_uio->uio_iov = oo->iov; 173 tag_uio->uio_iov[iovcnt].iov_base = trailer; 174 tag_uio->uio_iov[iovcnt].iov_len = AES_GMAC_HASH_LEN; 175 tag_uio->uio_iovcnt++; 176 crp->crp_digest_start = tag_uio->uio_resid; 177 tag_uio->uio_resid += AES_GMAC_HASH_LEN; 178 179 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 180 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 181 crypto_use_uio(crp, &uio); 182 if (!inplace) 183 crypto_use_output_uio(crp, &out_uio); 184 crp->crp_opaque = oo; 185 crp->crp_callback = ktls_ocf_callback; 186 187 counter_u64_add(ocf_tls12_gcm_crypts, 1); 188 if (inplace) 189 counter_u64_add(ocf_inplace, 1); 190 else 191 counter_u64_add(ocf_separate_output, 1); 192 for (;;) { 193 error = crypto_dispatch(crp); 194 if (error) 195 break; 196 197 mtx_lock(&os->lock); 198 while (!oo->done) 199 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 200 mtx_unlock(&os->lock); 201 202 if (crp->crp_etype != EAGAIN) { 203 error = crp->crp_etype; 204 break; 205 } 206 207 crp->crp_etype = 0; 208 crp->crp_flags &= ~CRYPTO_F_DONE; 209 oo->done = false; 210 counter_u64_add(ocf_retries, 1); 211 } 212 213 crypto_freereq(crp); 214 free(oo, M_KTLS_OCF); 215 return (error); 216 } 217 218 static int 219 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 220 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 221 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 222 { 223 struct uio uio, out_uio; 224 struct tls_aead_data_13 ad; 225 char nonce[12]; 226 struct cryptop *crp; 227 struct ocf_session *os; 228 struct ocf_operation *oo; 229 struct iovec *iov, *out_iov; 230 int i, error; 231 bool inplace; 232 233 os = tls->cipher; 234 235 oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(*iov) * 2, M_KTLS_OCF, 236 M_WAITOK | M_ZERO); 237 oo->os = os; 238 iov = oo->iov; 239 out_iov = iov + iovcnt + 2; 240 241 crp = crypto_getreq(os->sid, M_WAITOK); 242 243 /* Setup the nonce. */ 244 memcpy(nonce, tls->params.iv, tls->params.iv_len); 245 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 246 247 /* Setup the AAD. */ 248 ad.type = hdr->tls_type; 249 ad.tls_vmajor = hdr->tls_vmajor; 250 ad.tls_vminor = hdr->tls_vminor; 251 ad.tls_length = hdr->tls_length; 252 crp->crp_aad = &ad; 253 crp->crp_aad_length = sizeof(ad); 254 255 /* Compute payload length and determine if encryption is in place. */ 256 inplace = true; 257 crp->crp_payload_start = 0; 258 for (i = 0; i < iovcnt; i++) { 259 if (iniov[i].iov_base != outiov[i].iov_base) 260 inplace = false; 261 crp->crp_payload_length += iniov[i].iov_len; 262 } 263 264 /* Store the record type as the first byte of the trailer. */ 265 trailer[0] = record_type; 266 crp->crp_payload_length++; 267 crp->crp_digest_start = crp->crp_payload_length; 268 269 /* 270 * Duplicate the input iov to append the trailer. Always 271 * include the full trailer as input to get the record_type 272 * even if only the first byte is used. 273 */ 274 memcpy(iov, iniov, iovcnt * sizeof(*iov)); 275 iov[iovcnt].iov_base = trailer; 276 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1; 277 uio.uio_iov = iov; 278 uio.uio_iovcnt = iovcnt + 1; 279 uio.uio_offset = 0; 280 uio.uio_resid = crp->crp_payload_length + AES_GMAC_HASH_LEN; 281 uio.uio_segflg = UIO_SYSSPACE; 282 uio.uio_td = curthread; 283 crypto_use_uio(crp, &uio); 284 285 if (!inplace) { 286 /* Duplicate the output iov to append the trailer. */ 287 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov)); 288 out_iov[iovcnt] = iov[iovcnt]; 289 290 out_uio.uio_iov = out_iov; 291 out_uio.uio_iovcnt = iovcnt + 1; 292 out_uio.uio_offset = 0; 293 out_uio.uio_resid = crp->crp_payload_length + 294 AES_GMAC_HASH_LEN; 295 out_uio.uio_segflg = UIO_SYSSPACE; 296 out_uio.uio_td = curthread; 297 crypto_use_output_uio(crp, &out_uio); 298 } 299 300 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 301 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 302 crp->crp_opaque = oo; 303 crp->crp_callback = ktls_ocf_callback; 304 305 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 306 307 counter_u64_add(ocf_tls13_gcm_crypts, 1); 308 if (inplace) 309 counter_u64_add(ocf_inplace, 1); 310 else 311 counter_u64_add(ocf_separate_output, 1); 312 for (;;) { 313 error = crypto_dispatch(crp); 314 if (error) 315 break; 316 317 mtx_lock(&os->lock); 318 while (!oo->done) 319 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 320 mtx_unlock(&os->lock); 321 322 if (crp->crp_etype != EAGAIN) { 323 error = crp->crp_etype; 324 break; 325 } 326 327 crp->crp_etype = 0; 328 crp->crp_flags &= ~CRYPTO_F_DONE; 329 oo->done = false; 330 counter_u64_add(ocf_retries, 1); 331 } 332 333 crypto_freereq(crp); 334 free(oo, M_KTLS_OCF); 335 return (error); 336 } 337 338 static void 339 ktls_ocf_free(struct ktls_session *tls) 340 { 341 struct ocf_session *os; 342 343 os = tls->cipher; 344 crypto_freesession(os->sid); 345 mtx_destroy(&os->lock); 346 explicit_bzero(os, sizeof(*os)); 347 free(os, M_KTLS_OCF); 348 } 349 350 static int 351 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 352 { 353 struct crypto_session_params csp; 354 struct ocf_session *os; 355 int error; 356 357 memset(&csp, 0, sizeof(csp)); 358 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD; 359 360 switch (tls->params.cipher_algorithm) { 361 case CRYPTO_AES_NIST_GCM_16: 362 switch (tls->params.cipher_key_len) { 363 case 128 / 8: 364 case 256 / 8: 365 break; 366 default: 367 return (EINVAL); 368 } 369 csp.csp_mode = CSP_MODE_AEAD; 370 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 371 csp.csp_cipher_key = tls->params.cipher_key; 372 csp.csp_cipher_klen = tls->params.cipher_key_len; 373 csp.csp_ivlen = AES_GCM_IV_LEN; 374 break; 375 default: 376 return (EPROTONOSUPPORT); 377 } 378 379 /* Only TLS 1.2 and 1.3 are supported. */ 380 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 381 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 382 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 383 return (EPROTONOSUPPORT); 384 385 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 386 if (os == NULL) 387 return (ENOMEM); 388 389 error = crypto_newsession(&os->sid, &csp, 390 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 391 if (error) { 392 free(os, M_KTLS_OCF); 393 return (error); 394 } 395 396 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 397 tls->cipher = os; 398 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 399 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 400 else 401 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 402 tls->free = ktls_ocf_free; 403 return (0); 404 } 405 406 struct ktls_crypto_backend ocf_backend = { 407 .name = "OCF", 408 .prio = 5, 409 .api_version = KTLS_API_VERSION, 410 .try = ktls_ocf_try, 411 }; 412 413 static int 414 ktls_ocf_modevent(module_t mod, int what, void *arg) 415 { 416 int error; 417 418 switch (what) { 419 case MOD_LOAD: 420 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 421 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 422 ocf_inplace = counter_u64_alloc(M_WAITOK); 423 ocf_separate_output = counter_u64_alloc(M_WAITOK); 424 ocf_retries = counter_u64_alloc(M_WAITOK); 425 return (ktls_crypto_backend_register(&ocf_backend)); 426 case MOD_UNLOAD: 427 error = ktls_crypto_backend_deregister(&ocf_backend); 428 if (error) 429 return (error); 430 counter_u64_free(ocf_tls12_gcm_crypts); 431 counter_u64_free(ocf_tls13_gcm_crypts); 432 counter_u64_free(ocf_inplace); 433 counter_u64_free(ocf_separate_output); 434 counter_u64_free(ocf_retries); 435 return (0); 436 default: 437 return (EOPNOTSUPP); 438 } 439 } 440 441 static moduledata_t ktls_ocf_moduledata = { 442 "ktls_ocf", 443 ktls_ocf_modevent, 444 NULL 445 }; 446 447 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 448