1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/types.h> 31 #include <sys/errno.h> 32 33 #ifdef _KERNEL 34 #include <sys/libkern.h> 35 #include <sys/malloc.h> 36 #include <sys/sysctl.h> 37 #include <opencrypto/cryptodev.h> 38 #include <opencrypto/xform.h> 39 #endif 40 41 #include <sys/zio_crypt.h> 42 #include <sys/fs/zfs.h> 43 #include <sys/zio.h> 44 45 #include <sys/freebsd_crypto.h> 46 47 #define SHA512_HMAC_BLOCK_SIZE 128 48 49 static int crypt_sessions = 0; 50 SYSCTL_DECL(_vfs_zfs); 51 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 52 &crypt_sessions, 0, "Number of cryptographic sessions created"); 53 54 void 55 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 56 { 57 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 58 k_opad[SHA512_HMAC_BLOCK_SIZE], 59 key[SHA512_HMAC_BLOCK_SIZE]; 60 SHA512_CTX lctx; 61 int i; 62 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 63 64 /* 65 * This code is based on the similar code in geom/eli/g_eli_hmac.c 66 */ 67 memset(key, 0, sizeof (key)); 68 if (c_key->ck_length == 0) 69 /* do nothing */; 70 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 71 memcpy(key, c_key->ck_data, cl_bytes); 72 else { 73 /* 74 * If key is longer than 128 bytes reset it to 75 * key = SHA512(key). 76 */ 77 SHA512_Init(&lctx); 78 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 79 SHA512_Final(key, &lctx); 80 } 81 82 /* XOR key with ipad and opad values. */ 83 for (i = 0; i < sizeof (key); i++) { 84 k_ipad[i] = key[i] ^ 0x36; 85 k_opad[i] = key[i] ^ 0x5c; 86 } 87 memset(key, 0, sizeof (key)); 88 89 /* Start inner SHA512. */ 90 SHA512_Init(&ctx->innerctx); 91 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 92 memset(k_ipad, 0, sizeof (k_ipad)); 93 /* Start outer SHA512. */ 94 SHA512_Init(&ctx->outerctx); 95 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 96 memset(k_opad, 0, sizeof (k_opad)); 97 } 98 99 void 100 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 101 { 102 SHA512_Update(&ctx->innerctx, data, datasize); 103 } 104 105 void 106 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 107 { 108 uint8_t digest[SHA512_DIGEST_LENGTH]; 109 110 /* Complete inner hash */ 111 SHA512_Final(digest, &ctx->innerctx); 112 113 /* Complete outer hash */ 114 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 115 SHA512_Final(digest, &ctx->outerctx); 116 117 memset(ctx, 0, sizeof (*ctx)); 118 /* mdsize == 0 means "Give me the whole hash!" */ 119 if (mdsize == 0) 120 mdsize = SHA512_DIGEST_LENGTH; 121 memcpy(md, digest, mdsize); 122 memset(digest, 0, sizeof (digest)); 123 } 124 125 void 126 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 127 void *out_data, size_t out_data_size) 128 { 129 struct hmac_ctx ctx; 130 131 crypto_mac_init(&ctx, key); 132 crypto_mac_update(&ctx, in_data, in_data_size); 133 crypto_mac_final(&ctx, out_data, out_data_size); 134 } 135 136 static int 137 freebsd_zfs_crypt_done(struct cryptop *crp) 138 { 139 freebsd_crypt_session_t *ses; 140 141 ses = crp->crp_opaque; 142 mtx_lock(&ses->fs_lock); 143 ses->fs_done = true; 144 mtx_unlock(&ses->fs_lock); 145 wakeup(crp); 146 return (0); 147 } 148 149 static int 150 freebsd_zfs_crypt_done_sync(struct cryptop *crp) 151 { 152 153 return (0); 154 } 155 156 void 157 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 158 { 159 mtx_destroy(&sess->fs_lock); 160 crypto_freesession(sess->fs_sid); 161 memset(sess, 0, sizeof (*sess)); 162 } 163 164 static int 165 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 166 { 167 int error; 168 169 crp->crp_opaque = session; 170 for (;;) { 171 #if __FreeBSD_version < 1400004 172 boolean_t async = ((crypto_ses2caps(crp->crp_session) & 173 CRYPTOCAP_F_SYNC) == 0); 174 #else 175 boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session); 176 #endif 177 crp->crp_callback = async ? freebsd_zfs_crypt_done : 178 freebsd_zfs_crypt_done_sync; 179 error = crypto_dispatch(crp); 180 if (error == 0) { 181 if (async) { 182 mtx_lock(&session->fs_lock); 183 while (session->fs_done == false) { 184 msleep(crp, &session->fs_lock, 0, 185 "zfs_crypto", 0); 186 } 187 mtx_unlock(&session->fs_lock); 188 } 189 error = crp->crp_etype; 190 } 191 192 if (error == ENOMEM) { 193 pause("zcrnomem", 1); 194 } else if (error != EAGAIN) { 195 break; 196 } 197 crp->crp_etype = 0; 198 crp->crp_flags &= ~CRYPTO_F_DONE; 199 session->fs_done = false; 200 } 201 return (error); 202 } 203 static void 204 freebsd_crypt_uio_debug_log(boolean_t encrypt, 205 freebsd_crypt_session_t *input_sessionp, 206 const struct zio_crypt_info *c_info, 207 zfs_uio_t *data_uio, 208 crypto_key_t *key, 209 uint8_t *ivbuf, 210 size_t datalen, 211 size_t auth_len) 212 { 213 #ifdef FCRYPTO_DEBUG 214 struct cryptodesc *crd; 215 uint8_t *p = NULL; 216 size_t total = 0; 217 218 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, " 219 "%p, %u, %u)\n", 220 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 221 c_info->ci_algname, c_info->ci_crypt_type, 222 (unsigned int)c_info->ci_keylen, c_info->ci_name, 223 data_uio, key->ck_data, 224 (unsigned int)key->ck_length, 225 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 226 printf("\tkey = { "); 227 for (int i = 0; i < key->ck_length / 8; i++) { 228 uint8_t *b = (uint8_t *)key->ck_data; 229 printf("%02x ", b[i]); 230 } 231 printf("}\n"); 232 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 233 printf("\tiovec #%d: <%p, %u>\n", i, 234 zfs_uio_iovbase(data_uio, i), 235 (unsigned int)zfs_uio_iovlen(data_uio, i)); 236 total += zfs_uio_iovlen(data_uio, i); 237 } 238 zfs_uio_resid(data_uio) = total; 239 #endif 240 } 241 /* 242 * Create a new cryptographic session. This should 243 * happen every time the key changes (including when 244 * it's first loaded). 245 */ 246 int 247 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 248 const struct zio_crypt_info *c_info, crypto_key_t *key) 249 { 250 struct crypto_session_params csp = {0}; 251 int error = 0; 252 253 #ifdef FCRYPTO_DEBUG 254 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 255 __FUNCTION__, sessp, 256 c_info->ci_algname, c_info->ci_crypt_type, 257 (unsigned int)c_info->ci_keylen, c_info->ci_name, 258 key->ck_data, (unsigned int)key->ck_length); 259 printf("\tkey = { "); 260 for (int i = 0; i < key->ck_length / 8; i++) { 261 uint8_t *b = (uint8_t *)key->ck_data; 262 printf("%02x ", b[i]); 263 } 264 printf("}\n"); 265 #endif 266 csp.csp_mode = CSP_MODE_AEAD; 267 csp.csp_cipher_key = key->ck_data; 268 csp.csp_cipher_klen = key->ck_length / 8; 269 switch (c_info->ci_crypt_type) { 270 case ZC_TYPE_GCM: 271 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 272 csp.csp_ivlen = AES_GCM_IV_LEN; 273 switch (key->ck_length/8) { 274 case AES_128_GMAC_KEY_LEN: 275 case AES_192_GMAC_KEY_LEN: 276 case AES_256_GMAC_KEY_LEN: 277 break; 278 default: 279 error = EINVAL; 280 goto bad; 281 } 282 break; 283 case ZC_TYPE_CCM: 284 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 285 csp.csp_ivlen = AES_CCM_IV_LEN; 286 switch (key->ck_length/8) { 287 case AES_128_CBC_MAC_KEY_LEN: 288 case AES_192_CBC_MAC_KEY_LEN: 289 case AES_256_CBC_MAC_KEY_LEN: 290 break; 291 default: 292 error = EINVAL; 293 goto bad; 294 break; 295 } 296 break; 297 default: 298 error = ENOTSUP; 299 goto bad; 300 } 301 302 /* 303 * Disable the use of hardware drivers on FreeBSD 13 and later since 304 * common crypto offload drivers impose constraints on AES-GCM AAD 305 * lengths that make them unusable for ZFS, and we currently do not have 306 * a mechanism to fall back to a software driver for requests not 307 * handled by a hardware driver. 308 * 309 * On 12 we continue to permit the use of hardware drivers since 310 * CPU-accelerated drivers such as aesni(4) register themselves as 311 * hardware drivers. 312 */ 313 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 314 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 315 NULL, MTX_DEF); 316 crypt_sessions++; 317 bad: 318 #ifdef FCRYPTO_DEBUG 319 if (error) 320 printf("%s: returning error %d\n", __FUNCTION__, error); 321 #endif 322 return (error); 323 } 324 325 int 326 freebsd_crypt_uio(boolean_t encrypt, 327 freebsd_crypt_session_t *input_sessionp, 328 const struct zio_crypt_info *c_info, 329 zfs_uio_t *data_uio, 330 crypto_key_t *key, 331 uint8_t *ivbuf, 332 size_t datalen, 333 size_t auth_len) 334 { 335 struct cryptop *crp; 336 freebsd_crypt_session_t *session = NULL; 337 int error = 0; 338 size_t total = 0; 339 340 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 341 key, ivbuf, datalen, auth_len); 342 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 343 total += zfs_uio_iovlen(data_uio, i); 344 zfs_uio_resid(data_uio) = total; 345 if (input_sessionp == NULL) { 346 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 347 error = freebsd_crypt_newsession(session, c_info, key); 348 if (error) 349 goto out; 350 } else 351 session = input_sessionp; 352 353 crp = crypto_getreq(session->fs_sid, M_WAITOK); 354 if (encrypt) { 355 crp->crp_op = CRYPTO_OP_ENCRYPT | 356 CRYPTO_OP_COMPUTE_DIGEST; 357 } else { 358 crp->crp_op = CRYPTO_OP_DECRYPT | 359 CRYPTO_OP_VERIFY_DIGEST; 360 } 361 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 362 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 363 364 crp->crp_aad_start = 0; 365 crp->crp_aad_length = auth_len; 366 crp->crp_payload_start = auth_len; 367 crp->crp_payload_length = datalen; 368 crp->crp_digest_start = auth_len + datalen; 369 370 memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN); 371 error = zfs_crypto_dispatch(session, crp); 372 crypto_freereq(crp); 373 out: 374 #ifdef FCRYPTO_DEBUG 375 if (error) 376 printf("%s: returning error %d\n", __FUNCTION__, error); 377 #endif 378 if (input_sessionp == NULL) { 379 freebsd_crypt_freesession(session); 380 kmem_free(session, sizeof (*session)); 381 } 382 return (error); 383 } 384