1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 4 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 29 */ 30 31 #include <sys/types.h> 32 #include <sys/errno.h> 33 34 #ifdef _KERNEL 35 #include <sys/libkern.h> 36 #include <sys/malloc.h> 37 #include <sys/sysctl.h> 38 #include <opencrypto/cryptodev.h> 39 #include <opencrypto/xform.h> 40 #endif 41 42 #include <sys/zio_crypt.h> 43 #include <sys/fs/zfs.h> 44 #include <sys/zio.h> 45 46 #include <sys/freebsd_crypto.h> 47 48 #define SHA512_HMAC_BLOCK_SIZE 128 49 50 static int crypt_sessions = 0; 51 SYSCTL_DECL(_vfs_zfs); 52 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 53 &crypt_sessions, 0, "Number of cryptographic sessions created"); 54 55 void 56 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 57 { 58 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 59 k_opad[SHA512_HMAC_BLOCK_SIZE], 60 key[SHA512_HMAC_BLOCK_SIZE]; 61 SHA512_CTX lctx; 62 int i; 63 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 64 65 /* 66 * This code is based on the similar code in geom/eli/g_eli_hmac.c 67 */ 68 memset(key, 0, sizeof (key)); 69 if (c_key->ck_length == 0) 70 /* do nothing */; 71 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 72 memcpy(key, c_key->ck_data, cl_bytes); 73 else { 74 /* 75 * If key is longer than 128 bytes reset it to 76 * key = SHA512(key). 77 */ 78 SHA512_Init(&lctx); 79 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 80 SHA512_Final(key, &lctx); 81 } 82 83 /* XOR key with ipad and opad values. */ 84 for (i = 0; i < sizeof (key); i++) { 85 k_ipad[i] = key[i] ^ 0x36; 86 k_opad[i] = key[i] ^ 0x5c; 87 } 88 memset(key, 0, sizeof (key)); 89 90 /* Start inner SHA512. */ 91 SHA512_Init(&ctx->innerctx); 92 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 93 memset(k_ipad, 0, sizeof (k_ipad)); 94 /* Start outer SHA512. */ 95 SHA512_Init(&ctx->outerctx); 96 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 97 memset(k_opad, 0, sizeof (k_opad)); 98 } 99 100 void 101 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 102 { 103 SHA512_Update(&ctx->innerctx, data, datasize); 104 } 105 106 void 107 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 108 { 109 uint8_t digest[SHA512_DIGEST_LENGTH]; 110 111 /* Complete inner hash */ 112 SHA512_Final(digest, &ctx->innerctx); 113 114 /* Complete outer hash */ 115 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 116 SHA512_Final(digest, &ctx->outerctx); 117 118 memset(ctx, 0, sizeof (*ctx)); 119 /* mdsize == 0 means "Give me the whole hash!" */ 120 if (mdsize == 0) 121 mdsize = SHA512_DIGEST_LENGTH; 122 memcpy(md, digest, mdsize); 123 memset(digest, 0, sizeof (digest)); 124 } 125 126 void 127 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 128 void *out_data, size_t out_data_size) 129 { 130 struct hmac_ctx ctx; 131 132 crypto_mac_init(&ctx, key); 133 crypto_mac_update(&ctx, in_data, in_data_size); 134 crypto_mac_final(&ctx, out_data, out_data_size); 135 } 136 137 static int 138 freebsd_zfs_crypt_done(struct cryptop *crp) 139 { 140 freebsd_crypt_session_t *ses; 141 142 ses = crp->crp_opaque; 143 mtx_lock(&ses->fs_lock); 144 ses->fs_done = true; 145 mtx_unlock(&ses->fs_lock); 146 wakeup(crp); 147 return (0); 148 } 149 150 static int 151 freebsd_zfs_crypt_done_sync(struct cryptop *crp) 152 { 153 154 return (0); 155 } 156 157 void 158 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 159 { 160 mtx_destroy(&sess->fs_lock); 161 crypto_freesession(sess->fs_sid); 162 memset(sess, 0, sizeof (*sess)); 163 } 164 165 static int 166 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 167 { 168 int error; 169 170 crp->crp_opaque = session; 171 for (;;) { 172 #if __FreeBSD_version < 1400004 173 boolean_t async = ((crypto_ses2caps(crp->crp_session) & 174 CRYPTOCAP_F_SYNC) == 0); 175 #else 176 boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session); 177 #endif 178 crp->crp_callback = async ? freebsd_zfs_crypt_done : 179 freebsd_zfs_crypt_done_sync; 180 error = crypto_dispatch(crp); 181 if (error == 0) { 182 if (async) { 183 mtx_lock(&session->fs_lock); 184 while (session->fs_done == false) { 185 msleep(crp, &session->fs_lock, 0, 186 "zfs_crypto", 0); 187 } 188 mtx_unlock(&session->fs_lock); 189 } 190 error = crp->crp_etype; 191 } 192 193 if (error == ENOMEM) { 194 pause("zcrnomem", 1); 195 } else if (error != EAGAIN) { 196 break; 197 } 198 crp->crp_etype = 0; 199 crp->crp_flags &= ~CRYPTO_F_DONE; 200 session->fs_done = false; 201 } 202 return (error); 203 } 204 static void 205 freebsd_crypt_uio_debug_log(boolean_t encrypt, 206 freebsd_crypt_session_t *input_sessionp, 207 const struct zio_crypt_info *c_info, 208 zfs_uio_t *data_uio, 209 crypto_key_t *key, 210 uint8_t *ivbuf, 211 size_t datalen, 212 size_t auth_len) 213 { 214 #ifdef FCRYPTO_DEBUG 215 struct cryptodesc *crd; 216 uint8_t *p = NULL; 217 size_t total = 0; 218 219 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, " 220 "%p, %u, %u)\n", 221 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 222 c_info->ci_algname, c_info->ci_crypt_type, 223 (unsigned int)c_info->ci_keylen, c_info->ci_name, 224 data_uio, key->ck_data, 225 (unsigned int)key->ck_length, 226 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 227 printf("\tkey = { "); 228 for (int i = 0; i < key->ck_length / 8; i++) { 229 uint8_t *b = (uint8_t *)key->ck_data; 230 printf("%02x ", b[i]); 231 } 232 printf("}\n"); 233 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 234 printf("\tiovec #%d: <%p, %u>\n", i, 235 zfs_uio_iovbase(data_uio, i), 236 (unsigned int)zfs_uio_iovlen(data_uio, i)); 237 total += zfs_uio_iovlen(data_uio, i); 238 } 239 zfs_uio_resid(data_uio) = total; 240 #endif 241 } 242 /* 243 * Create a new cryptographic session. This should 244 * happen every time the key changes (including when 245 * it's first loaded). 246 */ 247 int 248 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 249 const struct zio_crypt_info *c_info, crypto_key_t *key) 250 { 251 struct crypto_session_params csp = {0}; 252 int error = 0; 253 254 #ifdef FCRYPTO_DEBUG 255 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 256 __FUNCTION__, sessp, 257 c_info->ci_algname, c_info->ci_crypt_type, 258 (unsigned int)c_info->ci_keylen, c_info->ci_name, 259 key->ck_data, (unsigned int)key->ck_length); 260 printf("\tkey = { "); 261 for (int i = 0; i < key->ck_length / 8; i++) { 262 uint8_t *b = (uint8_t *)key->ck_data; 263 printf("%02x ", b[i]); 264 } 265 printf("}\n"); 266 #endif 267 csp.csp_mode = CSP_MODE_AEAD; 268 csp.csp_cipher_key = key->ck_data; 269 csp.csp_cipher_klen = key->ck_length / 8; 270 switch (c_info->ci_crypt_type) { 271 case ZC_TYPE_GCM: 272 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 273 csp.csp_ivlen = AES_GCM_IV_LEN; 274 switch (key->ck_length/8) { 275 case AES_128_GMAC_KEY_LEN: 276 case AES_192_GMAC_KEY_LEN: 277 case AES_256_GMAC_KEY_LEN: 278 break; 279 default: 280 error = EINVAL; 281 goto bad; 282 } 283 break; 284 case ZC_TYPE_CCM: 285 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 286 csp.csp_ivlen = AES_CCM_IV_LEN; 287 switch (key->ck_length/8) { 288 case AES_128_CBC_MAC_KEY_LEN: 289 case AES_192_CBC_MAC_KEY_LEN: 290 case AES_256_CBC_MAC_KEY_LEN: 291 break; 292 default: 293 error = EINVAL; 294 goto bad; 295 break; 296 } 297 break; 298 default: 299 error = ENOTSUP; 300 goto bad; 301 } 302 303 /* 304 * Disable the use of hardware drivers on FreeBSD 13 and later since 305 * common crypto offload drivers impose constraints on AES-GCM AAD 306 * lengths that make them unusable for ZFS, and we currently do not have 307 * a mechanism to fall back to a software driver for requests not 308 * handled by a hardware driver. 309 * 310 * On 12 we continue to permit the use of hardware drivers since 311 * CPU-accelerated drivers such as aesni(4) register themselves as 312 * hardware drivers. 313 */ 314 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 315 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 316 NULL, MTX_DEF); 317 crypt_sessions++; 318 bad: 319 #ifdef FCRYPTO_DEBUG 320 if (error) 321 printf("%s: returning error %d\n", __FUNCTION__, error); 322 #endif 323 return (error); 324 } 325 326 int 327 freebsd_crypt_uio(boolean_t encrypt, 328 freebsd_crypt_session_t *input_sessionp, 329 const struct zio_crypt_info *c_info, 330 zfs_uio_t *data_uio, 331 crypto_key_t *key, 332 uint8_t *ivbuf, 333 size_t datalen, 334 size_t auth_len) 335 { 336 struct cryptop *crp; 337 freebsd_crypt_session_t *session = NULL; 338 int error = 0; 339 size_t total = 0; 340 341 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 342 key, ivbuf, datalen, auth_len); 343 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 344 total += zfs_uio_iovlen(data_uio, i); 345 zfs_uio_resid(data_uio) = total; 346 if (input_sessionp == NULL) { 347 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 348 error = freebsd_crypt_newsession(session, c_info, key); 349 if (error) 350 goto out; 351 } else 352 session = input_sessionp; 353 354 crp = crypto_getreq(session->fs_sid, M_WAITOK); 355 if (encrypt) { 356 crp->crp_op = CRYPTO_OP_ENCRYPT | 357 CRYPTO_OP_COMPUTE_DIGEST; 358 } else { 359 crp->crp_op = CRYPTO_OP_DECRYPT | 360 CRYPTO_OP_VERIFY_DIGEST; 361 } 362 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 363 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 364 365 crp->crp_aad_start = 0; 366 crp->crp_aad_length = auth_len; 367 crp->crp_payload_start = auth_len; 368 crp->crp_payload_length = datalen; 369 crp->crp_digest_start = auth_len + datalen; 370 371 memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN); 372 error = zfs_crypto_dispatch(session, crp); 373 crypto_freereq(crp); 374 out: 375 #ifdef FCRYPTO_DEBUG 376 if (error) 377 printf("%s: returning error %d\n", __FUNCTION__, error); 378 #endif 379 if (input_sessionp == NULL) { 380 freebsd_crypt_freesession(session); 381 kmem_free(session, sizeof (*session)); 382 } 383 return (error); 384 } 385