1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 struct mtx lock; 49 }; 50 51 struct ocf_operation { 52 struct ocf_session *os; 53 bool done; 54 struct iovec iov[0]; 55 }; 56 57 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 58 59 SYSCTL_DECL(_kern_ipc_tls); 60 SYSCTL_DECL(_kern_ipc_tls_stats); 61 62 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, 63 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 64 "Kernel TLS offload via OCF stats"); 65 66 static counter_u64_t ocf_tls12_gcm_crypts; 67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, 68 CTLFLAG_RD, &ocf_tls12_gcm_crypts, 69 "Total number of OCF TLS 1.2 GCM encryption operations"); 70 71 static counter_u64_t ocf_tls13_gcm_crypts; 72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, 73 CTLFLAG_RD, &ocf_tls13_gcm_crypts, 74 "Total number of OCF TLS 1.3 GCM encryption operations"); 75 76 static counter_u64_t ocf_retries; 77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD, 78 &ocf_retries, 79 "Number of OCF encryption operation retries"); 80 81 static int 82 ktls_ocf_callback(struct cryptop *crp) 83 { 84 struct ocf_operation *oo; 85 86 oo = crp->crp_opaque; 87 mtx_lock(&oo->os->lock); 88 oo->done = true; 89 mtx_unlock(&oo->os->lock); 90 wakeup(oo); 91 return (0); 92 } 93 94 static int 95 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, 96 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 97 struct iovec *outiov, int iovcnt, uint64_t seqno, 98 uint8_t record_type __unused) 99 { 100 struct uio uio; 101 struct tls_aead_data ad; 102 struct cryptop *crp; 103 struct ocf_session *os; 104 struct ocf_operation *oo; 105 struct iovec *iov; 106 int i, error; 107 uint16_t tls_comp_len; 108 109 os = tls->cipher; 110 111 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 112 M_WAITOK | M_ZERO); 113 oo->os = os; 114 iov = oo->iov; 115 116 crp = crypto_getreq(os->sid, M_WAITOK); 117 118 /* Setup the IV. */ 119 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); 120 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); 121 122 /* Setup the AAD. */ 123 tls_comp_len = ntohs(hdr->tls_length) - 124 (AES_GMAC_HASH_LEN + sizeof(uint64_t)); 125 ad.seq = htobe64(seqno); 126 ad.type = hdr->tls_type; 127 ad.tls_vmajor = hdr->tls_vmajor; 128 ad.tls_vminor = hdr->tls_vminor; 129 ad.tls_length = htons(tls_comp_len); 130 iov[0].iov_base = &ad; 131 iov[0].iov_len = sizeof(ad); 132 uio.uio_resid = sizeof(ad); 133 134 /* 135 * OCF always does encryption in place, so copy the data if 136 * needed. Ugh. 137 */ 138 for (i = 0; i < iovcnt; i++) { 139 iov[i + 1] = outiov[i]; 140 if (iniov[i].iov_base != outiov[i].iov_base) 141 memcpy(outiov[i].iov_base, iniov[i].iov_base, 142 outiov[i].iov_len); 143 uio.uio_resid += outiov[i].iov_len; 144 } 145 146 iov[iovcnt + 1].iov_base = trailer; 147 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN; 148 uio.uio_resid += AES_GMAC_HASH_LEN; 149 150 uio.uio_iov = iov; 151 uio.uio_iovcnt = iovcnt + 2; 152 uio.uio_offset = 0; 153 uio.uio_segflg = UIO_SYSSPACE; 154 uio.uio_td = curthread; 155 156 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 157 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 158 crp->crp_buf_type = CRYPTO_BUF_UIO; 159 crp->crp_uio = &uio; 160 crp->crp_ilen = uio.uio_resid; 161 crp->crp_opaque = oo; 162 crp->crp_callback = ktls_ocf_callback; 163 164 crp->crp_aad_start = 0; 165 crp->crp_aad_length = sizeof(ad); 166 crp->crp_payload_start = sizeof(ad); 167 crp->crp_payload_length = crp->crp_ilen - 168 (sizeof(ad) + AES_GMAC_HASH_LEN); 169 crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN; 170 171 counter_u64_add(ocf_tls12_gcm_crypts, 1); 172 for (;;) { 173 error = crypto_dispatch(crp); 174 if (error) 175 break; 176 177 mtx_lock(&os->lock); 178 while (!oo->done) 179 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 180 mtx_unlock(&os->lock); 181 182 if (crp->crp_etype != EAGAIN) { 183 error = crp->crp_etype; 184 break; 185 } 186 187 crp->crp_etype = 0; 188 crp->crp_flags &= ~CRYPTO_F_DONE; 189 oo->done = false; 190 counter_u64_add(ocf_retries, 1); 191 } 192 193 crypto_freereq(crp); 194 free(oo, M_KTLS_OCF); 195 return (error); 196 } 197 198 static int 199 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, 200 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, 201 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type) 202 { 203 struct uio uio; 204 struct tls_aead_data_13 ad; 205 char nonce[12]; 206 struct cryptop *crp; 207 struct ocf_session *os; 208 struct ocf_operation *oo; 209 struct iovec *iov; 210 int i, error; 211 212 os = tls->cipher; 213 214 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 215 M_WAITOK | M_ZERO); 216 oo->os = os; 217 iov = oo->iov; 218 219 crp = crypto_getreq(os->sid, M_WAITOK); 220 221 /* Setup the nonce. */ 222 memcpy(nonce, tls->params.iv, tls->params.iv_len); 223 *(uint64_t *)(nonce + 4) ^= htobe64(seqno); 224 225 /* Setup the AAD. */ 226 ad.type = hdr->tls_type; 227 ad.tls_vmajor = hdr->tls_vmajor; 228 ad.tls_vminor = hdr->tls_vminor; 229 ad.tls_length = hdr->tls_length; 230 iov[0].iov_base = &ad; 231 iov[0].iov_len = sizeof(ad); 232 uio.uio_resid = sizeof(ad); 233 234 /* 235 * OCF always does encryption in place, so copy the data if 236 * needed. Ugh. 237 */ 238 for (i = 0; i < iovcnt; i++) { 239 iov[i + 1] = outiov[i]; 240 if (iniov[i].iov_base != outiov[i].iov_base) 241 memcpy(outiov[i].iov_base, iniov[i].iov_base, 242 outiov[i].iov_len); 243 uio.uio_resid += outiov[i].iov_len; 244 } 245 246 trailer[0] = record_type; 247 iov[iovcnt + 1].iov_base = trailer; 248 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1; 249 uio.uio_resid += AES_GMAC_HASH_LEN + 1; 250 251 uio.uio_iov = iov; 252 uio.uio_iovcnt = iovcnt + 2; 253 uio.uio_offset = 0; 254 uio.uio_segflg = UIO_SYSSPACE; 255 uio.uio_td = curthread; 256 257 crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; 258 crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; 259 crp->crp_buf_type = CRYPTO_BUF_UIO; 260 crp->crp_uio = &uio; 261 crp->crp_ilen = uio.uio_resid; 262 crp->crp_opaque = oo; 263 crp->crp_callback = ktls_ocf_callback; 264 265 crp->crp_aad_start = 0; 266 crp->crp_aad_length = sizeof(ad); 267 crp->crp_payload_start = sizeof(ad); 268 crp->crp_payload_length = crp->crp_ilen - 269 (sizeof(ad) + AES_GMAC_HASH_LEN); 270 crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN; 271 memcpy(crp->crp_iv, nonce, sizeof(nonce)); 272 273 counter_u64_add(ocf_tls13_gcm_crypts, 1); 274 for (;;) { 275 error = crypto_dispatch(crp); 276 if (error) 277 break; 278 279 mtx_lock(&os->lock); 280 while (!oo->done) 281 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 282 mtx_unlock(&os->lock); 283 284 if (crp->crp_etype != EAGAIN) { 285 error = crp->crp_etype; 286 break; 287 } 288 289 crp->crp_etype = 0; 290 crp->crp_flags &= ~CRYPTO_F_DONE; 291 oo->done = false; 292 counter_u64_add(ocf_retries, 1); 293 } 294 295 crypto_freereq(crp); 296 free(oo, M_KTLS_OCF); 297 return (error); 298 } 299 300 static void 301 ktls_ocf_free(struct ktls_session *tls) 302 { 303 struct ocf_session *os; 304 305 os = tls->cipher; 306 crypto_freesession(os->sid); 307 mtx_destroy(&os->lock); 308 explicit_bzero(os, sizeof(*os)); 309 free(os, M_KTLS_OCF); 310 } 311 312 static int 313 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 314 { 315 struct crypto_session_params csp; 316 struct ocf_session *os; 317 int error; 318 319 memset(&csp, 0, sizeof(csp)); 320 321 switch (tls->params.cipher_algorithm) { 322 case CRYPTO_AES_NIST_GCM_16: 323 switch (tls->params.cipher_key_len) { 324 case 128 / 8: 325 case 256 / 8: 326 break; 327 default: 328 return (EINVAL); 329 } 330 csp.csp_mode = CSP_MODE_AEAD; 331 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 332 csp.csp_cipher_key = tls->params.cipher_key; 333 csp.csp_cipher_klen = tls->params.cipher_key_len; 334 csp.csp_ivlen = AES_GCM_IV_LEN; 335 break; 336 default: 337 return (EPROTONOSUPPORT); 338 } 339 340 /* Only TLS 1.2 and 1.3 are supported. */ 341 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 342 tls->params.tls_vminor < TLS_MINOR_VER_TWO || 343 tls->params.tls_vminor > TLS_MINOR_VER_THREE) 344 return (EPROTONOSUPPORT); 345 346 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 347 if (os == NULL) 348 return (ENOMEM); 349 350 error = crypto_newsession(&os->sid, &csp, 351 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 352 if (error) { 353 free(os, M_KTLS_OCF); 354 return (error); 355 } 356 357 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 358 tls->cipher = os; 359 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) 360 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt; 361 else 362 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt; 363 tls->free = ktls_ocf_free; 364 return (0); 365 } 366 367 struct ktls_crypto_backend ocf_backend = { 368 .name = "OCF", 369 .prio = 5, 370 .api_version = KTLS_API_VERSION, 371 .try = ktls_ocf_try, 372 }; 373 374 static int 375 ktls_ocf_modevent(module_t mod, int what, void *arg) 376 { 377 int error; 378 379 switch (what) { 380 case MOD_LOAD: 381 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK); 382 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK); 383 ocf_retries = counter_u64_alloc(M_WAITOK); 384 return (ktls_crypto_backend_register(&ocf_backend)); 385 case MOD_UNLOAD: 386 error = ktls_crypto_backend_deregister(&ocf_backend); 387 if (error) 388 return (error); 389 counter_u64_free(ocf_tls12_gcm_crypts); 390 counter_u64_free(ocf_tls13_gcm_crypts); 391 counter_u64_free(ocf_retries); 392 return (0); 393 default: 394 return (EOPNOTSUPP); 395 } 396 } 397 398 static moduledata_t ktls_ocf_moduledata = { 399 "ktls_ocf", 400 ktls_ocf_modevent, 401 NULL 402 }; 403 404 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 405