1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 int crda_alg; 49 struct mtx lock; 50 }; 51 52 struct ocf_operation { 53 struct ocf_session *os; 54 bool done; 55 struct iovec iov[0]; 56 }; 57 58 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 59 60 SYSCTL_DECL(_kern_ipc_tls); 61 SYSCTL_DECL(_kern_ipc_tls_stats); 62 63 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 64 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 65 "Kernel TLS offload via OCF stats"); 66 67 static counter_u64_t ocf_tls12_gcm_crypts; 68 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 69 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 70 "Total number of OCF TLS 1.2 GCM encryption operations"); 71 72 static counter_u64_t ocf_tls13_gcm_crypts; 73 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 74 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 75 "Total number of OCF TLS 1.3 GCM encryption operations"); 76 77 static counter_u64_t ocf_retries; 78 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 79 &ocf_retries, 80 "Number of OCF encryption operation retries"); 81 82 static int 83 ktls_ocf_callback(struct cryptop *crp) 84 { 85 struct ocf_operation *oo; 86 87 oo = crp->crp_opaque; 88 mtx_lock(&oo->os->lock); 89 oo->done = true; 90 mtx_unlock(&oo->os->lock); 91 wakeup(oo); 92 return (0); 93 } 94 95 static int 96 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 97 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 98 struct iovec *outiov, int iovcnt, uint64_t seqno, 99 uint8_t record_type __unused) 100 { 101 struct uio uio; 102 struct tls_aead_data ad; 103 struct tls_nonce_data nd; 104 struct cryptodesc *crde, *crda; 105 struct cryptop *crp; 106 struct ocf_session *os; 107 struct ocf_operation *oo; 108 struct iovec *iov; 109 int i, error; 110 uint16_t tls_comp_len; 111 112 os = tls->cipher; 113 114 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 115 M_WAITOK | M_ZERO); 116 oo->os = os; 117 iov = oo->iov; 118 119 crp = crypto_getreq(2); 120 if (crp == NULL) { 121 free(oo, M_KTLS_OCF); 122 return (ENOMEM); 123 } 124 125 /* Setup the IV. */ 126 memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN); 127 memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); 128 129 /* Setup the AAD. */ 130 tls_comp_len = ntohs(hdr->tls_length) - 131 (AES_GMAC_HASH_LEN + sizeof(nd.seq)); 132 ad.seq = htobe64(seqno); 133 ad.type = hdr->tls_type; 134 ad.tls_vmajor = hdr->tls_vmajor; 135 ad.tls_vminor = hdr->tls_vminor; 136 ad.tls_length = htons(tls_comp_len); 137 iov[0].iov_base = &ad; 138 iov[0].iov_len = sizeof(ad); 139 uio.uio_resid = sizeof(ad); 140 141 /* 142 * OCF always does encryption in place, so copy the data if 143 * needed. Ugh. 144 */ 145 for (i = 0; i < iovcnt; i++) { 146 iov[i + 1] = outiov[i]; 147 if (iniov[i].iov_base != outiov[i].iov_base) 148 memcpy(outiov[i].iov_base, iniov[i].iov_base, 149 outiov[i].iov_len); 150 uio.uio_resid += outiov[i].iov_len; 151 } 152 153 iov[iovcnt + 1].iov_base = trailer; 154 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN; 155 uio.uio_resid += AES_GMAC_HASH_LEN; 156 157 uio.uio_iov = iov; 158 uio.uio_iovcnt = iovcnt + 2; 159 uio.uio_offset = 0; 160 uio.uio_segflg = UIO_SYSSPACE; 161 uio.uio_td = curthread; 162 163 crp->crp_session = os->sid; 164 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; 165 crp->crp_uio = &uio; 166 crp->crp_ilen = uio.uio_resid; 167 crp->crp_opaque = oo; 168 crp->crp_callback = ktls_ocf_callback; 169 170 crde = crp->crp_desc; 171 crda = crde->crd_next; 172 173 crda->crd_alg = os->crda_alg; 174 crda->crd_skip = 0; 175 crda->crd_len = sizeof(ad); 176 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; 177 178 crde->crd_alg = CRYPTO_AES_NIST_GCM_16; 179 crde->crd_skip = sizeof(ad); 180 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); 181 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 182 memcpy(crde->crd_iv, &nd, sizeof(nd)); 183 184 counter_u64_add(ocf_tls12_gcm_crypts, 1); 185 for (;;) { 186 error = crypto_dispatch(crp); 187 if (error) 188 break; 189 190 mtx_lock(&os->lock); 191 while (!oo->done) 192 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 193 mtx_unlock(&os->lock); 194 195 if (crp->crp_etype != EAGAIN) { 196 error = crp->crp_etype; 197 break; 198 } 199 200 crp->crp_etype = 0; 201 crp->crp_flags &= ~CRYPTO_F_DONE; 202 oo->done = false; 203 counter_u64_add(ocf_retries, 1); 204 } 205 206 crypto_freereq(crp); 207 free(oo, M_KTLS_OCF); 208 return (error); 209 } 210 211 static int 212 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 213 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 214 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 215 { 216 struct uio uio; 217 struct tls_aead_data_13 ad; 218 char nonce[12]; 219 struct cryptodesc *crde, *crda; 220 struct cryptop *crp; 221 struct ocf_session *os; 222 struct ocf_operation *oo; 223 struct iovec *iov; 224 int i, error; 225 226 os = tls->cipher; 227 228 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 229 M_WAITOK | M_ZERO); 230 oo->os = os; 231 iov = oo->iov; 232 233 crp = crypto_getreq(2); 234 if (crp == NULL) { 235 free(oo, M_KTLS_OCF); 236 return (ENOMEM); 237 } 238 239 /* Setup the nonce. */ 240 memcpy(nonce, tls->params.iv, tls->params.iv_len); 241 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 242 243 /* Setup the AAD. */ 244 ad.type = hdr->tls_type; 245 ad.tls_vmajor = hdr->tls_vmajor; 246 ad.tls_vminor = hdr->tls_vminor; 247 ad.tls_length = hdr->tls_length; 248 iov[0].iov_base = &ad; 249 iov[0].iov_len = sizeof(ad); 250 uio.uio_resid = sizeof(ad); 251 252 /* 253 * OCF always does encryption in place, so copy the data if 254 * needed. Ugh. 255 */ 256 for (i = 0; i < iovcnt; i++) { 257 iov[i + 1] = outiov[i]; 258 if (iniov[i].iov_base != outiov[i].iov_base) 259 memcpy(outiov[i].iov_base, iniov[i].iov_base, 260 outiov[i].iov_len); 261 uio.uio_resid += outiov[i].iov_len; 262 } 263 264 trailer[0] = record_type; 265 iov[iovcnt + 1].iov_base = trailer; 266 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1; 267 uio.uio_resid += AES_GMAC_HASH_LEN + 1; 268 269 uio.uio_iov = iov; 270 uio.uio_iovcnt = iovcnt + 2; 271 uio.uio_offset = 0; 272 uio.uio_segflg = UIO_SYSSPACE; 273 uio.uio_td = curthread; 274 275 crp->crp_session = os->sid; 276 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; 277 crp->crp_uio = &uio; 278 crp->crp_ilen = uio.uio_resid; 279 crp->crp_opaque = oo; 280 crp->crp_callback = ktls_ocf_callback; 281 282 crde = crp->crp_desc; 283 crda = crde->crd_next; 284 285 crda->crd_alg = os->crda_alg; 286 crda->crd_skip = 0; 287 crda->crd_len = sizeof(ad); 288 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; 289 290 crde->crd_alg = CRYPTO_AES_NIST_GCM_16; 291 crde->crd_skip = sizeof(ad); 292 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); 293 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 294 memcpy(crde->crd_iv, nonce, sizeof(nonce)); 295 296 counter_u64_add(ocf_tls13_gcm_crypts, 1); 297 for (;;) { 298 error = crypto_dispatch(crp); 299 if (error) 300 break; 301 302 mtx_lock(&os->lock); 303 while (!oo->done) 304 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 305 mtx_unlock(&os->lock); 306 307 if (crp->crp_etype != EAGAIN) { 308 error = crp->crp_etype; 309 break; 310 } 311 312 crp->crp_etype = 0; 313 crp->crp_flags &= ~CRYPTO_F_DONE; 314 oo->done = false; 315 counter_u64_add(ocf_retries, 1); 316 } 317 318 crypto_freereq(crp); 319 free(oo, M_KTLS_OCF); 320 return (error); 321 } 322 323 static void 324 ktls_ocf_free(struct ktls_session *tls) 325 { 326 struct ocf_session *os; 327 328 os = tls->cipher; 329 mtx_destroy(&os->lock); 330 explicit_bzero(os, sizeof(*os)); 331 free(os, M_KTLS_OCF); 332 } 333 334 static int 335 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 336 { 337 struct cryptoini cria, crie; 338 struct ocf_session *os; 339 int error; 340 341 memset(&cria, 0, sizeof(cria)); 342 memset(&crie, 0, sizeof(crie)); 343 344 switch (tls->params.cipher_algorithm) { 345 case CRYPTO_AES_NIST_GCM_16: 346 switch (tls->params.cipher_key_len) { 347 case 128 / 8: 348 cria.cri_alg = CRYPTO_AES_128_NIST_GMAC; 349 break; 350 case 256 / 8: 351 cria.cri_alg = CRYPTO_AES_256_NIST_GMAC; 352 break; 353 default: 354 return (EINVAL); 355 } 356 cria.cri_key = tls->params.cipher_key; 357 cria.cri_klen = tls->params.cipher_key_len * 8; 358 break; 359 default: 360 return (EPROTONOSUPPORT); 361 } 362 363 /* Only TLS 1.2 and 1.3 are supported. */ 364 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 365 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 366 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 367 return (EPROTONOSUPPORT); 368 369 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 370 if (os == NULL) 371 return (ENOMEM); 372 373 crie.cri_alg = tls->params.cipher_algorithm; 374 crie.cri_key = tls->params.cipher_key; 375 crie.cri_klen = tls->params.cipher_key_len * 8; 376 377 crie.cri_next = &cria; 378 error = crypto_newsession(&os->sid, &crie, 379 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 380 if (error) { 381 free(os, M_KTLS_OCF); 382 return (error); 383 } 384 385 os->crda_alg = cria.cri_alg; 386 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 387 tls->cipher = os; 388 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 389 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 390 else 391 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 392 tls->free = ktls_ocf_free; 393 return (0); 394 } 395 396 struct ktls_crypto_backend ocf_backend = { 397 .name = "OCF", 398 .prio = 5, 399 .api_version = KTLS_API_VERSION, 400 .try = ktls_ocf_try, 401 }; 402 403 static int 404 ktls_ocf_modevent(module_t mod, int what, void *arg) 405 { 406 int error; 407 408 switch (what) { 409 case MOD_LOAD: 410 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 411 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 412 ocf_retries = counter_u64_alloc(M_WAITOK); 413 return (ktls_crypto_backend_register(&ocf_backend)); 414 case MOD_UNLOAD: 415 error = ktls_crypto_backend_deregister(&ocf_backend); 416 if (error) 417 return (error); 418 counter_u64_free(ocf_tls12_gcm_crypts); 419 counter_u64_free(ocf_tls13_gcm_crypts); 420 counter_u64_free(ocf_retries); 421 return (0); 422 default: 423 return (EOPNOTSUPP); 424 } 425 } 426 427 static moduledata_t ktls_ocf_moduledata = { 428 "ktls_ocf", 429 ktls_ocf_modevent, 430 NULL 431 }; 432 433 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 434