1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 int crda_alg; 49 struct mtx lock; 50 }; 51 52 struct ocf_operation { 53 struct ocf_session *os; 54 bool done; 55 struct iovec iov[0]; 56 }; 57 58 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 59 60 SYSCTL_DECL(_kern_ipc_tls); 61 SYSCTL_DECL(_kern_ipc_tls_stats); 62 63 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, CTLFLAG_RD, 0, 64 "Kernel TLS offload via OCF stats"); 65 66 static counter_u64_t ocf_tls12_gcm_crypts; 67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 68 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 69 "Total number of OCF TLS 1.2 GCM encryption operations"); 70 71 static counter_u64_t ocf_tls13_gcm_crypts; 72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 73 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 74 "Total number of OCF TLS 1.3 GCM encryption operations"); 75 76 static counter_u64_t ocf_retries; 77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 78 &ocf_retries, 79 "Number of OCF encryption operation retries"); 80 81 static int 82 ktls_ocf_callback(struct cryptop *crp) 83 { 84 struct ocf_operation *oo; 85 86 oo = crp->crp_opaque; 87 mtx_lock(&oo->os->lock); 88 oo->done = true; 89 mtx_unlock(&oo->os->lock); 90 wakeup(oo); 91 return (0); 92 } 93 94 static int 95 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 96 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 97 struct iovec *outiov, int iovcnt, uint64_t seqno, 98 uint8_t record_type __unused) 99 { 100 struct uio uio; 101 struct tls_aead_data ad; 102 struct tls_nonce_data nd; 103 struct cryptodesc *crde, *crda; 104 struct cryptop *crp; 105 struct ocf_session *os; 106 struct ocf_operation *oo; 107 struct iovec *iov; 108 int i, error; 109 uint16_t tls_comp_len; 110 111 os = tls->cipher; 112 113 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 114 M_WAITOK | M_ZERO); 115 oo->os = os; 116 iov = oo->iov; 117 118 crp = crypto_getreq(2); 119 if (crp == NULL) { 120 free(oo, M_KTLS_OCF); 121 return (ENOMEM); 122 } 123 124 /* Setup the IV. */ 125 memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN); 126 memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); 127 128 /* Setup the AAD. */ 129 tls_comp_len = ntohs(hdr->tls_length) - 130 (AES_GMAC_HASH_LEN + sizeof(nd.seq)); 131 ad.seq = htobe64(seqno); 132 ad.type = hdr->tls_type; 133 ad.tls_vmajor = hdr->tls_vmajor; 134 ad.tls_vminor = hdr->tls_vminor; 135 ad.tls_length = htons(tls_comp_len); 136 iov[0].iov_base = &ad; 137 iov[0].iov_len = sizeof(ad); 138 uio.uio_resid = sizeof(ad); 139 140 /* 141 * OCF always does encryption in place, so copy the data if 142 * needed. Ugh. 143 */ 144 for (i = 0; i < iovcnt; i++) { 145 iov[i + 1] = outiov[i]; 146 if (iniov[i].iov_base != outiov[i].iov_base) 147 memcpy(outiov[i].iov_base, iniov[i].iov_base, 148 outiov[i].iov_len); 149 uio.uio_resid += outiov[i].iov_len; 150 } 151 152 iov[iovcnt + 1].iov_base = trailer; 153 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN; 154 uio.uio_resid += AES_GMAC_HASH_LEN; 155 156 uio.uio_iov = iov; 157 uio.uio_iovcnt = iovcnt + 2; 158 uio.uio_offset = 0; 159 uio.uio_segflg = UIO_SYSSPACE; 160 uio.uio_td = curthread; 161 162 crp->crp_session = os->sid; 163 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; 164 crp->crp_uio = &uio; 165 crp->crp_ilen = uio.uio_resid; 166 crp->crp_opaque = oo; 167 crp->crp_callback = ktls_ocf_callback; 168 169 crde = crp->crp_desc; 170 crda = crde->crd_next; 171 172 crda->crd_alg = os->crda_alg; 173 crda->crd_skip = 0; 174 crda->crd_len = sizeof(ad); 175 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; 176 177 crde->crd_alg = CRYPTO_AES_NIST_GCM_16; 178 crde->crd_skip = sizeof(ad); 179 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); 180 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 181 memcpy(crde->crd_iv, &nd, sizeof(nd)); 182 183 counter_u64_add(ocf_tls12_gcm_crypts, 1); 184 for (;;) { 185 error = crypto_dispatch(crp); 186 if (error) 187 break; 188 189 mtx_lock(&os->lock); 190 while (!oo->done) 191 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 192 mtx_unlock(&os->lock); 193 194 if (crp->crp_etype != EAGAIN) { 195 error = crp->crp_etype; 196 break; 197 } 198 199 crp->crp_etype = 0; 200 crp->crp_flags &= ~CRYPTO_F_DONE; 201 oo->done = false; 202 counter_u64_add(ocf_retries, 1); 203 } 204 205 crypto_freereq(crp); 206 free(oo, M_KTLS_OCF); 207 return (error); 208 } 209 210 static int 211 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 212 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 213 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 214 { 215 struct uio uio; 216 struct tls_aead_data_13 ad; 217 char nonce[12]; 218 struct cryptodesc *crde, *crda; 219 struct cryptop *crp; 220 struct ocf_session *os; 221 struct ocf_operation *oo; 222 struct iovec *iov; 223 int i, error; 224 225 os = tls->cipher; 226 227 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 228 M_WAITOK | M_ZERO); 229 oo->os = os; 230 iov = oo->iov; 231 232 crp = crypto_getreq(2); 233 if (crp == NULL) { 234 free(oo, M_KTLS_OCF); 235 return (ENOMEM); 236 } 237 238 /* Setup the nonce. */ 239 memcpy(nonce, tls->params.iv, tls->params.iv_len); 240 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 241 242 /* Setup the AAD. */ 243 ad.type = hdr->tls_type; 244 ad.tls_vmajor = hdr->tls_vmajor; 245 ad.tls_vminor = hdr->tls_vminor; 246 ad.tls_length = hdr->tls_length; 247 iov[0].iov_base = &ad; 248 iov[0].iov_len = sizeof(ad); 249 uio.uio_resid = sizeof(ad); 250 251 /* 252 * OCF always does encryption in place, so copy the data if 253 * needed. Ugh. 254 */ 255 for (i = 0; i < iovcnt; i++) { 256 iov[i + 1] = outiov[i]; 257 if (iniov[i].iov_base != outiov[i].iov_base) 258 memcpy(outiov[i].iov_base, iniov[i].iov_base, 259 outiov[i].iov_len); 260 uio.uio_resid += outiov[i].iov_len; 261 } 262 263 trailer[0] = record_type; 264 iov[iovcnt + 1].iov_base = trailer; 265 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1; 266 uio.uio_resid += AES_GMAC_HASH_LEN + 1; 267 268 uio.uio_iov = iov; 269 uio.uio_iovcnt = iovcnt + 2; 270 uio.uio_offset = 0; 271 uio.uio_segflg = UIO_SYSSPACE; 272 uio.uio_td = curthread; 273 274 crp->crp_session = os->sid; 275 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; 276 crp->crp_uio = &uio; 277 crp->crp_ilen = uio.uio_resid; 278 crp->crp_opaque = oo; 279 crp->crp_callback = ktls_ocf_callback; 280 281 crde = crp->crp_desc; 282 crda = crde->crd_next; 283 284 crda->crd_alg = os->crda_alg; 285 crda->crd_skip = 0; 286 crda->crd_len = sizeof(ad); 287 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; 288 289 crde->crd_alg = CRYPTO_AES_NIST_GCM_16; 290 crde->crd_skip = sizeof(ad); 291 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); 292 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 293 memcpy(crde->crd_iv, nonce, sizeof(nonce)); 294 295 counter_u64_add(ocf_tls13_gcm_crypts, 1); 296 for (;;) { 297 error = crypto_dispatch(crp); 298 if (error) 299 break; 300 301 mtx_lock(&os->lock); 302 while (!oo->done) 303 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 304 mtx_unlock(&os->lock); 305 306 if (crp->crp_etype != EAGAIN) { 307 error = crp->crp_etype; 308 break; 309 } 310 311 crp->crp_etype = 0; 312 crp->crp_flags &= ~CRYPTO_F_DONE; 313 oo->done = false; 314 counter_u64_add(ocf_retries, 1); 315 } 316 317 crypto_freereq(crp); 318 free(oo, M_KTLS_OCF); 319 return (error); 320 } 321 322 static void 323 ktls_ocf_free(struct ktls_session *tls) 324 { 325 struct ocf_session *os; 326 327 os = tls->cipher; 328 mtx_destroy(&os->lock); 329 explicit_bzero(os, sizeof(*os)); 330 free(os, M_KTLS_OCF); 331 } 332 333 static int 334 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 335 { 336 struct cryptoini cria, crie; 337 struct ocf_session *os; 338 int error; 339 340 memset(&cria, 0, sizeof(cria)); 341 memset(&crie, 0, sizeof(crie)); 342 343 switch (tls->params.cipher_algorithm) { 344 case CRYPTO_AES_NIST_GCM_16: 345 switch (tls->params.cipher_key_len) { 346 case 128 / 8: 347 cria.cri_alg = CRYPTO_AES_128_NIST_GMAC; 348 break; 349 case 256 / 8: 350 cria.cri_alg = CRYPTO_AES_256_NIST_GMAC; 351 break; 352 default: 353 return (EINVAL); 354 } 355 cria.cri_key = tls->params.cipher_key; 356 cria.cri_klen = tls->params.cipher_key_len * 8; 357 break; 358 default: 359 return (EPROTONOSUPPORT); 360 } 361 362 /* Only TLS 1.2 and 1.3 are supported. */ 363 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 364 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 365 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 366 return (EPROTONOSUPPORT); 367 368 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 369 if (os == NULL) 370 return (ENOMEM); 371 372 crie.cri_alg = tls->params.cipher_algorithm; 373 crie.cri_key = tls->params.cipher_key; 374 crie.cri_klen = tls->params.cipher_key_len * 8; 375 376 crie.cri_next = &cria; 377 error = crypto_newsession(&os->sid, &crie, 378 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 379 if (error) { 380 free(os, M_KTLS_OCF); 381 return (error); 382 } 383 384 os->crda_alg = cria.cri_alg; 385 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 386 tls->cipher = os; 387 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 388 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 389 else 390 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 391 tls->free = ktls_ocf_free; 392 return (0); 393 } 394 395 struct ktls_crypto_backend ocf_backend = { 396 .name = "OCF", 397 .prio = 5, 398 .api_version = KTLS_API_VERSION, 399 .try = ktls_ocf_try, 400 }; 401 402 static int 403 ktls_ocf_modevent(module_t mod, int what, void *arg) 404 { 405 int error; 406 407 switch (what) { 408 case MOD_LOAD: 409 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 410 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 411 ocf_retries = counter_u64_alloc(M_WAITOK); 412 return (ktls_crypto_backend_register(&ocf_backend)); 413 case MOD_UNLOAD: 414 error = ktls_crypto_backend_deregister(&ocf_backend); 415 if (error) 416 return (error); 417 counter_u64_free(ocf_tls12_gcm_crypts); 418 counter_u64_free(ocf_tls13_gcm_crypts); 419 counter_u64_free(ocf_retries); 420 return (0); 421 default: 422 return (EOPNOTSUPP); 423 } 424 } 425 426 static moduledata_t ktls_ocf_moduledata = { 427 "ktls_ocf", 428 ktls_ocf_modevent, 429 NULL 430 }; 431 432 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 433