1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/types.h> 34 #include <sys/errno.h> 35 36 #ifdef _KERNEL 37 #include <sys/libkern.h> 38 #include <sys/malloc.h> 39 #include <sys/sysctl.h> 40 #include <opencrypto/cryptodev.h> 41 #include <opencrypto/xform.h> 42 #endif 43 44 #include <sys/zio_crypt.h> 45 #include <sys/fs/zfs.h> 46 #include <sys/zio.h> 47 48 #include <sys/freebsd_crypto.h> 49 50 #define SHA512_HMAC_BLOCK_SIZE 128 51 52 static int crypt_sessions = 0; 53 SYSCTL_DECL(_vfs_zfs); 54 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 55 &crypt_sessions, 0, "Number of cryptographic sessions created"); 56 57 void 58 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 59 { 60 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 61 k_opad[SHA512_HMAC_BLOCK_SIZE], 62 key[SHA512_HMAC_BLOCK_SIZE]; 63 SHA512_CTX lctx; 64 int i; 65 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 66 67 /* 68 * This code is based on the similar code in geom/eli/g_eli_hmac.c 69 */ 70 memset(key, 0, sizeof (key)); 71 if (c_key->ck_length == 0) 72 /* do nothing */; 73 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 74 memcpy(key, c_key->ck_data, cl_bytes); 75 else { 76 /* 77 * If key is longer than 128 bytes reset it to 78 * key = SHA512(key). 79 */ 80 SHA512_Init(&lctx); 81 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 82 SHA512_Final(key, &lctx); 83 } 84 85 /* XOR key with ipad and opad values. */ 86 for (i = 0; i < sizeof (key); i++) { 87 k_ipad[i] = key[i] ^ 0x36; 88 k_opad[i] = key[i] ^ 0x5c; 89 } 90 memset(key, 0, sizeof (key)); 91 92 /* Start inner SHA512. */ 93 SHA512_Init(&ctx->innerctx); 94 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 95 memset(k_ipad, 0, sizeof (k_ipad)); 96 /* Start outer SHA512. */ 97 SHA512_Init(&ctx->outerctx); 98 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 99 memset(k_opad, 0, sizeof (k_opad)); 100 } 101 102 void 103 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 104 { 105 SHA512_Update(&ctx->innerctx, data, datasize); 106 } 107 108 void 109 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 110 { 111 uint8_t digest[SHA512_DIGEST_LENGTH]; 112 113 /* Complete inner hash */ 114 SHA512_Final(digest, &ctx->innerctx); 115 116 /* Complete outer hash */ 117 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 118 SHA512_Final(digest, &ctx->outerctx); 119 120 memset(ctx, 0, sizeof (*ctx)); 121 /* mdsize == 0 means "Give me the whole hash!" */ 122 if (mdsize == 0) 123 mdsize = SHA512_DIGEST_LENGTH; 124 memcpy(md, digest, mdsize); 125 memset(digest, 0, sizeof (digest)); 126 } 127 128 void 129 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 130 void *out_data, size_t out_data_size) 131 { 132 struct hmac_ctx ctx; 133 134 crypto_mac_init(&ctx, key); 135 crypto_mac_update(&ctx, in_data, in_data_size); 136 crypto_mac_final(&ctx, out_data, out_data_size); 137 } 138 139 static int 140 freebsd_zfs_crypt_done(struct cryptop *crp) 141 { 142 freebsd_crypt_session_t *ses; 143 144 ses = crp->crp_opaque; 145 mtx_lock(&ses->fs_lock); 146 ses->fs_done = true; 147 mtx_unlock(&ses->fs_lock); 148 wakeup(crp); 149 return (0); 150 } 151 152 void 153 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 154 { 155 mtx_destroy(&sess->fs_lock); 156 crypto_freesession(sess->fs_sid); 157 memset(sess, 0, sizeof (*sess)); 158 } 159 160 static int 161 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 162 { 163 int error; 164 165 crp->crp_opaque = session; 166 crp->crp_callback = freebsd_zfs_crypt_done; 167 for (;;) { 168 error = crypto_dispatch(crp); 169 if (error) 170 break; 171 mtx_lock(&session->fs_lock); 172 while (session->fs_done == false) 173 msleep(crp, &session->fs_lock, 0, 174 "zfs_crypto", 0); 175 mtx_unlock(&session->fs_lock); 176 177 if (crp->crp_etype == ENOMEM) { 178 pause("zcrnomem", 1); 179 } else if (crp->crp_etype != EAGAIN) { 180 error = crp->crp_etype; 181 break; 182 } 183 crp->crp_etype = 0; 184 crp->crp_flags &= ~CRYPTO_F_DONE; 185 session->fs_done = false; 186 #if __FreeBSD_version < 1300087 187 /* 188 * Session ID changed, so we should record that, 189 * and try again 190 */ 191 session->fs_sid = crp->crp_session; 192 #endif 193 } 194 return (error); 195 } 196 static void 197 freebsd_crypt_uio_debug_log(boolean_t encrypt, 198 freebsd_crypt_session_t *input_sessionp, 199 const struct zio_crypt_info *c_info, 200 zfs_uio_t *data_uio, 201 crypto_key_t *key, 202 uint8_t *ivbuf, 203 size_t datalen, 204 size_t auth_len) 205 { 206 #ifdef FCRYPTO_DEBUG 207 struct cryptodesc *crd; 208 uint8_t *p = NULL; 209 size_t total = 0; 210 211 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, " 212 "%p, %u, %u)\n", 213 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 214 c_info->ci_algname, c_info->ci_crypt_type, 215 (unsigned int)c_info->ci_keylen, c_info->ci_name, 216 data_uio, key->ck_data, 217 (unsigned int)key->ck_length, 218 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 219 printf("\tkey = { "); 220 for (int i = 0; i < key->ck_length / 8; i++) { 221 uint8_t *b = (uint8_t *)key->ck_data; 222 printf("%02x ", b[i]); 223 } 224 printf("}\n"); 225 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 226 printf("\tiovec #%d: <%p, %u>\n", i, 227 zfs_uio_iovbase(data_uio, i), 228 (unsigned int)zfs_uio_iovlen(data_uio, i)); 229 total += zfs_uio_iovlen(data_uio, i); 230 } 231 zfs_uio_resid(data_uio) = total; 232 #endif 233 } 234 /* 235 * Create a new cryptographic session. This should 236 * happen every time the key changes (including when 237 * it's first loaded). 238 */ 239 #if __FreeBSD_version >= 1300087 240 int 241 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 242 const struct zio_crypt_info *c_info, crypto_key_t *key) 243 { 244 struct crypto_session_params csp = {0}; 245 int error = 0; 246 247 #ifdef FCRYPTO_DEBUG 248 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 249 __FUNCTION__, sessp, 250 c_info->ci_algname, c_info->ci_crypt_type, 251 (unsigned int)c_info->ci_keylen, c_info->ci_name, 252 key->ck_data, (unsigned int)key->ck_length); 253 printf("\tkey = { "); 254 for (int i = 0; i < key->ck_length / 8; i++) { 255 uint8_t *b = (uint8_t *)key->ck_data; 256 printf("%02x ", b[i]); 257 } 258 printf("}\n"); 259 #endif 260 csp.csp_mode = CSP_MODE_AEAD; 261 csp.csp_cipher_key = key->ck_data; 262 csp.csp_cipher_klen = key->ck_length / 8; 263 switch (c_info->ci_crypt_type) { 264 case ZC_TYPE_GCM: 265 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 266 csp.csp_ivlen = AES_GCM_IV_LEN; 267 switch (key->ck_length/8) { 268 case AES_128_GMAC_KEY_LEN: 269 case AES_192_GMAC_KEY_LEN: 270 case AES_256_GMAC_KEY_LEN: 271 break; 272 default: 273 error = EINVAL; 274 goto bad; 275 } 276 break; 277 case ZC_TYPE_CCM: 278 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 279 csp.csp_ivlen = AES_CCM_IV_LEN; 280 switch (key->ck_length/8) { 281 case AES_128_CBC_MAC_KEY_LEN: 282 case AES_192_CBC_MAC_KEY_LEN: 283 case AES_256_CBC_MAC_KEY_LEN: 284 break; 285 default: 286 error = EINVAL; 287 goto bad; 288 break; 289 } 290 break; 291 default: 292 error = ENOTSUP; 293 goto bad; 294 } 295 296 /* 297 * Disable the use of hardware drivers on FreeBSD 13 and later since 298 * common crypto offload drivers impose constraints on AES-GCM AAD 299 * lengths that make them unusable for ZFS, and we currently do not have 300 * a mechanism to fall back to a software driver for requests not 301 * handled by a hardware driver. 302 * 303 * On 12 we continue to permit the use of hardware drivers since 304 * CPU-accelerated drivers such as aesni(4) register themselves as 305 * hardware drivers. 306 */ 307 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 308 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 309 NULL, MTX_DEF); 310 crypt_sessions++; 311 bad: 312 #ifdef FCRYPTO_DEBUG 313 if (error) 314 printf("%s: returning error %d\n", __FUNCTION__, error); 315 #endif 316 return (error); 317 } 318 319 int 320 freebsd_crypt_uio(boolean_t encrypt, 321 freebsd_crypt_session_t *input_sessionp, 322 const struct zio_crypt_info *c_info, 323 zfs_uio_t *data_uio, 324 crypto_key_t *key, 325 uint8_t *ivbuf, 326 size_t datalen, 327 size_t auth_len) 328 { 329 struct cryptop *crp; 330 freebsd_crypt_session_t *session = NULL; 331 int error = 0; 332 size_t total = 0; 333 334 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 335 key, ivbuf, datalen, auth_len); 336 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 337 total += zfs_uio_iovlen(data_uio, i); 338 zfs_uio_resid(data_uio) = total; 339 if (input_sessionp == NULL) { 340 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 341 error = freebsd_crypt_newsession(session, c_info, key); 342 if (error) 343 goto out; 344 } else 345 session = input_sessionp; 346 347 crp = crypto_getreq(session->fs_sid, M_WAITOK); 348 if (encrypt) { 349 crp->crp_op = CRYPTO_OP_ENCRYPT | 350 CRYPTO_OP_COMPUTE_DIGEST; 351 } else { 352 crp->crp_op = CRYPTO_OP_DECRYPT | 353 CRYPTO_OP_VERIFY_DIGEST; 354 } 355 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 356 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 357 358 crp->crp_aad_start = 0; 359 crp->crp_aad_length = auth_len; 360 crp->crp_payload_start = auth_len; 361 crp->crp_payload_length = datalen; 362 crp->crp_digest_start = auth_len + datalen; 363 364 memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN); 365 error = zfs_crypto_dispatch(session, crp); 366 crypto_freereq(crp); 367 out: 368 #ifdef FCRYPTO_DEBUG 369 if (error) 370 printf("%s: returning error %d\n", __FUNCTION__, error); 371 #endif 372 if (input_sessionp == NULL) { 373 freebsd_crypt_freesession(session); 374 kmem_free(session, sizeof (*session)); 375 } 376 return (error); 377 } 378 379 #else 380 int 381 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 382 const struct zio_crypt_info *c_info, crypto_key_t *key) 383 { 384 struct cryptoini cria = {0}, crie = {0}, *crip; 385 struct enc_xform *xform; 386 struct auth_hash *xauth; 387 int error = 0; 388 crypto_session_t sid; 389 390 #ifdef FCRYPTO_DEBUG 391 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 392 __FUNCTION__, sessp, 393 c_info->ci_algname, c_info->ci_crypt_type, 394 (unsigned int)c_info->ci_keylen, c_info->ci_name, 395 key->ck_data, (unsigned int)key->ck_length); 396 printf("\tkey = { "); 397 for (int i = 0; i < key->ck_length / 8; i++) { 398 uint8_t *b = (uint8_t *)key->ck_data; 399 printf("%02x ", b[i]); 400 } 401 printf("}\n"); 402 #endif 403 switch (c_info->ci_crypt_type) { 404 case ZC_TYPE_GCM: 405 xform = &enc_xform_aes_nist_gcm; 406 switch (key->ck_length/8) { 407 case AES_128_GMAC_KEY_LEN: 408 xauth = &auth_hash_nist_gmac_aes_128; 409 break; 410 case AES_192_GMAC_KEY_LEN: 411 xauth = &auth_hash_nist_gmac_aes_192; 412 break; 413 case AES_256_GMAC_KEY_LEN: 414 xauth = &auth_hash_nist_gmac_aes_256; 415 break; 416 default: 417 error = EINVAL; 418 goto bad; 419 } 420 break; 421 case ZC_TYPE_CCM: 422 xform = &enc_xform_ccm; 423 switch (key->ck_length/8) { 424 case AES_128_CBC_MAC_KEY_LEN: 425 xauth = &auth_hash_ccm_cbc_mac_128; 426 break; 427 case AES_192_CBC_MAC_KEY_LEN: 428 xauth = &auth_hash_ccm_cbc_mac_192; 429 break; 430 case AES_256_CBC_MAC_KEY_LEN: 431 xauth = &auth_hash_ccm_cbc_mac_256; 432 break; 433 default: 434 error = EINVAL; 435 goto bad; 436 break; 437 } 438 break; 439 default: 440 error = ENOTSUP; 441 goto bad; 442 } 443 #ifdef FCRYPTO_DEBUG 444 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 445 "auth %s (key length %d)\n", 446 __FUNCTION__, __LINE__, 447 xform->name, (unsigned int)key->ck_length, 448 (unsigned int)key->ck_length/8, 449 xauth->name, xauth->keysize); 450 #endif 451 452 crie.cri_alg = xform->type; 453 crie.cri_key = key->ck_data; 454 crie.cri_klen = key->ck_length; 455 456 cria.cri_alg = xauth->type; 457 cria.cri_key = key->ck_data; 458 cria.cri_klen = key->ck_length; 459 460 cria.cri_next = &crie; 461 crie.cri_next = NULL; 462 crip = &cria; 463 // Everything else is zero-initialised 464 465 error = crypto_newsession(&sid, crip, 466 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 467 if (error != 0) { 468 printf("%s(%d): crypto_newsession failed with %d\n", 469 __FUNCTION__, __LINE__, error); 470 goto bad; 471 } 472 sessp->fs_sid = sid; 473 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 474 NULL, MTX_DEF); 475 crypt_sessions++; 476 bad: 477 return (error); 478 } 479 480 /* 481 * The meat of encryption/decryption. 482 * If sessp is NULL, then it will create a 483 * temporary cryptographic session, and release 484 * it when done. 485 */ 486 int 487 freebsd_crypt_uio(boolean_t encrypt, 488 freebsd_crypt_session_t *input_sessionp, 489 const struct zio_crypt_info *c_info, 490 zfs_uio_t *data_uio, 491 crypto_key_t *key, 492 uint8_t *ivbuf, 493 size_t datalen, 494 size_t auth_len) 495 { 496 struct cryptop *crp; 497 struct cryptodesc *enc_desc, *auth_desc; 498 struct enc_xform *xform; 499 struct auth_hash *xauth; 500 freebsd_crypt_session_t *session = NULL; 501 int error; 502 503 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 504 key, ivbuf, datalen, auth_len); 505 switch (c_info->ci_crypt_type) { 506 case ZC_TYPE_GCM: 507 xform = &enc_xform_aes_nist_gcm; 508 switch (key->ck_length/8) { 509 case AES_128_GMAC_KEY_LEN: 510 xauth = &auth_hash_nist_gmac_aes_128; 511 break; 512 case AES_192_GMAC_KEY_LEN: 513 xauth = &auth_hash_nist_gmac_aes_192; 514 break; 515 case AES_256_GMAC_KEY_LEN: 516 xauth = &auth_hash_nist_gmac_aes_256; 517 break; 518 default: 519 error = EINVAL; 520 goto bad; 521 } 522 break; 523 case ZC_TYPE_CCM: 524 xform = &enc_xform_ccm; 525 switch (key->ck_length/8) { 526 case AES_128_CBC_MAC_KEY_LEN: 527 xauth = &auth_hash_ccm_cbc_mac_128; 528 break; 529 case AES_192_CBC_MAC_KEY_LEN: 530 xauth = &auth_hash_ccm_cbc_mac_192; 531 break; 532 case AES_256_CBC_MAC_KEY_LEN: 533 xauth = &auth_hash_ccm_cbc_mac_256; 534 break; 535 default: 536 error = EINVAL; 537 goto bad; 538 break; 539 } 540 break; 541 default: 542 error = ENOTSUP; 543 goto bad; 544 } 545 546 #ifdef FCRYPTO_DEBUG 547 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 548 "auth %s (key length %d)\n", 549 __FUNCTION__, __LINE__, 550 xform->name, (unsigned int)key->ck_length, 551 (unsigned int)key->ck_length/8, 552 xauth->name, xauth->keysize); 553 #endif 554 555 if (input_sessionp == NULL) { 556 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 557 error = freebsd_crypt_newsession(session, c_info, key); 558 if (error) 559 goto out; 560 } else 561 session = input_sessionp; 562 563 crp = crypto_getreq(2); 564 if (crp == NULL) { 565 error = ENOMEM; 566 goto bad; 567 } 568 569 auth_desc = crp->crp_desc; 570 enc_desc = auth_desc->crd_next; 571 572 crp->crp_session = session->fs_sid; 573 crp->crp_ilen = auth_len + datalen; 574 crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio); 575 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; 576 577 auth_desc->crd_skip = 0; 578 auth_desc->crd_len = auth_len; 579 auth_desc->crd_inject = auth_len + datalen; 580 auth_desc->crd_alg = xauth->type; 581 #ifdef FCRYPTO_DEBUG 582 printf("%s: auth: skip = %u, len = %u, inject = %u\n", 583 __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, 584 auth_desc->crd_inject); 585 #endif 586 587 enc_desc->crd_skip = auth_len; 588 enc_desc->crd_len = datalen; 589 enc_desc->crd_inject = auth_len; 590 enc_desc->crd_alg = xform->type; 591 enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 592 memcpy(enc_desc->crd_iv, ivbuf, ZIO_DATA_IV_LEN); 593 enc_desc->crd_next = NULL; 594 595 #ifdef FCRYPTO_DEBUG 596 printf("%s: enc: skip = %u, len = %u, inject = %u\n", 597 __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, 598 enc_desc->crd_inject); 599 #endif 600 601 if (encrypt) 602 enc_desc->crd_flags |= CRD_F_ENCRYPT; 603 604 error = zfs_crypto_dispatch(session, crp); 605 crypto_freereq(crp); 606 out: 607 if (input_sessionp == NULL) { 608 freebsd_crypt_freesession(session); 609 kmem_free(session, sizeof (*session)); 610 } 611 bad: 612 #ifdef FCRYPTO_DEBUG 613 if (error) 614 printf("%s: returning error %d\n", __FUNCTION__, error); 615 #endif 616 return (error); 617 } 618 #endif 619