1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Netflix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/counter.h> 35 #include <sys/endian.h> 36 #include <sys/kernel.h> 37 #include <sys/ktls.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/sysctl.h> 43 #include <sys/uio.h> 44 #include <opencrypto/cryptodev.h> 45 46 struct ocf_session { 47 crypto_session_t sid; 48 int crda_alg; 49 struct mtx lock; 50 }; 51 52 struct ocf_operation { 53 struct ocf_session *os; 54 bool done; 55 struct iovec iov[0]; 56 }; 57 58 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); 59 60 SYSCTL_DECL(_kern_ipc_tls); 61 SYSCTL_DECL(_kern_ipc_tls_stats); 62 63 static counter_u64_t ocf_gcm_crypts; 64 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_gcm_crypts, CTLFLAG_RD, 65 &ocf_gcm_crypts, 66 "Total number of OCF GCM encryption operations"); 67 68 static counter_u64_t ocf_retries; 69 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_retries, CTLFLAG_RD, 70 &ocf_retries, 71 "Number of OCF encryption operation retries"); 72 73 static int 74 ktls_ocf_callback(struct cryptop *crp) 75 { 76 struct ocf_operation *oo; 77 78 oo = crp->crp_opaque; 79 mtx_lock(&oo->os->lock); 80 oo->done = true; 81 mtx_unlock(&oo->os->lock); 82 wakeup(oo); 83 return (0); 84 } 85 86 static int 87 ktls_ocf_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr, 88 uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt, 89 uint64_t seqno, uint8_t record_type __unused) 90 { 91 struct uio uio; 92 struct tls_aead_data ad; 93 struct tls_nonce_data nd; 94 struct cryptodesc *crde, *crda; 95 struct cryptop *crp; 96 struct ocf_session *os; 97 struct ocf_operation *oo; 98 struct iovec *iov; 99 int i, error; 100 uint16_t tls_comp_len; 101 102 os = tls->cipher; 103 104 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF, 105 M_WAITOK | M_ZERO); 106 oo->os = os; 107 iov = oo->iov; 108 109 crp = crypto_getreq(2); 110 if (crp == NULL) { 111 free(oo, M_KTLS_OCF); 112 return (ENOMEM); 113 } 114 115 /* Setup the IV. */ 116 memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN); 117 memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); 118 119 /* Setup the AAD. */ 120 tls_comp_len = ntohs(hdr->tls_length) - 121 (AES_GMAC_HASH_LEN + sizeof(nd.seq)); 122 ad.seq = htobe64(seqno); 123 ad.type = hdr->tls_type; 124 ad.tls_vmajor = hdr->tls_vmajor; 125 ad.tls_vminor = hdr->tls_vminor; 126 ad.tls_length = htons(tls_comp_len); 127 iov[0].iov_base = &ad; 128 iov[0].iov_len = sizeof(ad); 129 uio.uio_resid = sizeof(ad); 130 131 /* 132 * OCF always does encryption in place, so copy the data if 133 * needed. Ugh. 134 */ 135 for (i = 0; i < iovcnt; i++) { 136 iov[i + 1] = outiov[i]; 137 if (iniov[i].iov_base != outiov[i].iov_base) 138 memcpy(outiov[i].iov_base, iniov[i].iov_base, 139 outiov[i].iov_len); 140 uio.uio_resid += outiov[i].iov_len; 141 } 142 143 iov[iovcnt + 1].iov_base = trailer; 144 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN; 145 uio.uio_resid += AES_GMAC_HASH_LEN; 146 147 uio.uio_iov = iov; 148 uio.uio_iovcnt = iovcnt + 2; 149 uio.uio_offset = 0; 150 uio.uio_segflg = UIO_SYSSPACE; 151 uio.uio_td = curthread; 152 153 crp->crp_session = os->sid; 154 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; 155 crp->crp_uio = &uio; 156 crp->crp_ilen = uio.uio_resid; 157 crp->crp_opaque = oo; 158 crp->crp_callback = ktls_ocf_callback; 159 160 crde = crp->crp_desc; 161 crda = crde->crd_next; 162 163 crda->crd_alg = os->crda_alg; 164 crda->crd_skip = 0; 165 crda->crd_len = sizeof(ad); 166 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; 167 168 crde->crd_alg = CRYPTO_AES_NIST_GCM_16; 169 crde->crd_skip = sizeof(ad); 170 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); 171 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 172 memcpy(crde->crd_iv, &nd, sizeof(nd)); 173 174 counter_u64_add(ocf_gcm_crypts, 1); 175 for (;;) { 176 error = crypto_dispatch(crp); 177 if (error) 178 break; 179 180 mtx_lock(&os->lock); 181 while (!oo->done) 182 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0); 183 mtx_unlock(&os->lock); 184 185 if (crp->crp_etype != EAGAIN) { 186 error = crp->crp_etype; 187 break; 188 } 189 190 crp->crp_etype = 0; 191 crp->crp_flags &= ~CRYPTO_F_DONE; 192 oo->done = false; 193 counter_u64_add(ocf_retries, 1); 194 } 195 196 crypto_freereq(crp); 197 free(oo, M_KTLS_OCF); 198 return (error); 199 } 200 201 static void 202 ktls_ocf_free(struct ktls_session *tls) 203 { 204 struct ocf_session *os; 205 206 os = tls->cipher; 207 mtx_destroy(&os->lock); 208 explicit_bzero(os, sizeof(*os)); 209 free(os, M_KTLS_OCF); 210 } 211 212 static int 213 ktls_ocf_try(struct socket *so, struct ktls_session *tls) 214 { 215 struct cryptoini cria, crie; 216 struct ocf_session *os; 217 int error; 218 219 memset(&cria, 0, sizeof(cria)); 220 memset(&crie, 0, sizeof(crie)); 221 222 switch (tls->params.cipher_algorithm) { 223 case CRYPTO_AES_NIST_GCM_16: 224 if (tls->params.iv_len != TLS_AEAD_GCM_LEN) 225 return (EINVAL); 226 switch (tls->params.cipher_key_len) { 227 case 128 / 8: 228 cria.cri_alg = CRYPTO_AES_128_NIST_GMAC; 229 break; 230 case 256 / 8: 231 cria.cri_alg = CRYPTO_AES_256_NIST_GMAC; 232 break; 233 default: 234 return (EINVAL); 235 } 236 cria.cri_key = tls->params.cipher_key; 237 cria.cri_klen = tls->params.cipher_key_len * 8; 238 break; 239 default: 240 return (EPROTONOSUPPORT); 241 } 242 243 /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 244 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 245 tls->params.tls_vminor < TLS_MINOR_VER_ONE || 246 tls->params.tls_vminor > TLS_MINOR_VER_TWO) 247 return (EPROTONOSUPPORT); 248 249 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO); 250 if (os == NULL) 251 return (ENOMEM); 252 253 crie.cri_alg = tls->params.cipher_algorithm; 254 crie.cri_key = tls->params.cipher_key; 255 crie.cri_klen = tls->params.cipher_key_len * 8; 256 257 crie.cri_next = &cria; 258 error = crypto_newsession(&os->sid, &crie, 259 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); 260 if (error) { 261 free(os, M_KTLS_OCF); 262 return (error); 263 } 264 265 os->crda_alg = cria.cri_alg; 266 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); 267 tls->cipher = os; 268 tls->sw_encrypt = ktls_ocf_encrypt; 269 tls->free = ktls_ocf_free; 270 return (0); 271 } 272 273 struct ktls_crypto_backend ocf_backend = { 274 .name = "OCF", 275 .prio = 5, 276 .api_version = KTLS_API_VERSION, 277 .try = ktls_ocf_try, 278 }; 279 280 static int 281 ktls_ocf_modevent(module_t mod, int what, void *arg) 282 { 283 int error; 284 285 switch (what) { 286 case MOD_LOAD: 287 ocf_gcm_crypts = counter_u64_alloc(M_WAITOK); 288 ocf_retries = counter_u64_alloc(M_WAITOK); 289 return (ktls_crypto_backend_register(&ocf_backend)); 290 case MOD_UNLOAD: 291 error = ktls_crypto_backend_deregister(&ocf_backend); 292 if (error) 293 return (error); 294 counter_u64_free(ocf_gcm_crypts); 295 counter_u64_free(ocf_retries); 296 return (0); 297 default: 298 return (EOPNOTSUPP); 299 } 300 } 301 302 static moduledata_t ktls_ocf_moduledata = { 303 "ktls_ocf", 304 ktls_ocf_modevent, 305 NULL 306 }; 307 308 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); 309